diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..b0d84b17a8dafc67a1fda72daa5291fbf4f25a21 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +# Tracked in camstack/server as hf/camstack-trainer/Dockerfile; edit it there, not on the Hub. +# +# The CamStack trainer image. Two builders read this file: +# - `docker build --platform linux/amd64 --build-arg TRAINER_REVISION= .` (any registry); +# - the hub's Modal provider, which builds it WITHOUT a build context: +# FROM becomes `images.fromRegistry(...)` and every other line one +# `dockerfileCommands` entry. +# So, load-bearing: +# - only FROM, ARG, ENV, RUN and WORKDIR — never COPY or ADD (there is no context); +# - one instruction per line, no backslash continuations (the hub splits on lines); +# - every pip version pinned; the trainer itself installed from this repository +# at TRAINER_REVISION (the hub passes the exact commit it was released with). +FROM pytorch/pytorch:2.13.0-cuda12.6-cudnn9-runtime +ARG TRAINER_REVISION=main +ENV PYTHONUNBUFFERED=1 +ENV PIP_NO_CACHE_DIR=1 +ENV TRAINER_REVISION=${TRAINER_REVISION} +ENV YOLO_CONFIG_DIR=/tmp/ultralytics +RUN apt-get update && apt-get install -y --no-install-recommends git libgl1 libglib2.0-0 && rm -rf /var/lib/apt/lists/* +RUN pip install "camstack-train[train,export] @ git+https://huggingface.co/camstack/camstack-trainer@${TRAINER_REVISION}" +RUN python -c "import camstack_train, ultralytics, openvino, nncf, coremltools; print('trainer', camstack_train.__version__)" +WORKDIR /app +CMD ["python", "-m", "camstack_train", "--help"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..2ed6a4f6d33b6dd262598b6dd207fd91e495199a --- /dev/null +++ b/LICENSE @@ -0,0 +1,666 @@ +Source: https://www.gnu.org/licenses/agpl-3.0.txt +Fetched: 2026-09-26 (verbatim below the line; nothing edited) +Applies to: Ultralytics weights: objectDetection/yolo26, objectDetection/yolov9, segmentation/yolo26-seg +Note: byte-identical to https://raw.githubusercontent.com/ultralytics/ultralytics/main/LICENSE, fetched the same day. +-------------------------------------------------------------------------------- + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4f8307a5c9769dfedd468c24fccd3ac401c29eed --- /dev/null +++ b/README.md @@ -0,0 +1,96 @@ +--- +license: agpl-3.0 +tags: + - object-detection + - yolov9 + - training + - camstack +library_name: ultralytics +--- + + + +# CamStack trainer + +The container a [CamStack](https://camstack-docs.zentik.app) hub runs on a +cloud GPU (Modal first) to fine-tune its YOLOv9 root detector on the operator's +own annotated frames. It evaluates the result against the model the cameras +run today, proposes per-class confidence floors, and exports every format a +CamStack node runs (ONNX, OpenVINO FP16 and INT8, Core ML FP16). + +**Licence: AGPL-3.0.** It imports Ultralytics (AGPL-3.0), and a model it +fine-tunes is an Ultralytics derivative. The hub never imports this package: it +talks to the container through files and a process boundary only. This +repository, Dockerfile included, is the Corresponding Source of every image +built from it. + +## The contract + +In: + +- `/data/job.json` — schema `camstack.training/v1`, every field filled by the + hub ([`schemas/job.schema.json`](schemas/job.schema.json)). An unknown schema + exits 64. +- `/data/dataset.tar` — the hub's annotated export + (`camstack.retrain-export/v2|v3`), verified against `dataset.sha256`. +- `/cache` — a persistent volume: base weights, the COCO replay slice, the + baseline ONNX. + +Out, in `/data/out`: + +- `bundle.tar` — every artefact under its flat catalog name: + `.pt`, `.onnx`, `-fp16.xml/.bin`, `-int8.xml/.bin`, + `.mlpackage/`. +- `result.json` — schema `camstack.training-result/v1` + ([`schemas/result.schema.json`](schemas/result.schema.json)): the dataset + split, training summary, baseline vs candidate evaluation, proposed floors, + the INT8 gate, one row per bundled file with its sha256, and a + `catalogDraft` (a `ModelCatalogEntry` per variant, relative URLs, revisions). + Written on failure too. + +Progress: one `CAMSTACK_EVENT {json}` line per event on stdout — `stage`, +`epoch`, `metric`, `artifact`, `error`, and a final `done` carrying the sha256 +of `result.json`. Every event has `v: 1` and `ts` (epoch ms). + +Exit codes: 0 ok · 64 bad job · 65 dataset unusable · 70 internal. + +## What it does + +1. **Prepare** — verify the archive, convert it to a YOLO layout (COCO-80 ids; + a vehicle/animal box needs a COCO `label` or its frame is left out of + training; `model_error` boxes are learned as background), and split it by + whole cameras or whole days, never by random frame. +2. **Replay** — mix a pinned slice of COCO val2017 back in so the 80-class head + is not forgotten. +3. **Train** — Ultralytics `YOLO(.pt).train(...)` with the job's + epochs / batch / lr0 / freeze / patience / seed. +4. **Export** — ONNX (static, opset 13), OpenVINO FP16, OpenVINO INT8 + calibrated on the user's own train frames letterboxed exactly like the + runtime, Core ML FP16. A failing optional format is dropped with its + reason; ONNX failing fails the job. +5. **Evaluate** — the CamStack evaluation harness (vendored from + `scripts/eval/`) runs the baseline and the candidate with the inference + pool's own preprocess and decode on the complete holdout frames, and the + D739 rule proposes floors. INT8 is dropped when its mAP50 falls more than + `int8Gate.maxMap50Drop` below FP16. +6. **Package** — `bundle.tar` + `result.json`. + +Logs carry counts only — never a frame path or a camera name. + +## Running it yourself + +```bash +docker build --platform linux/amd64 --build-arg TRAINER_REVISION=main -t camstack-trainer . +docker run --gpus all -v $PWD/data:/data -v $PWD/cache:/cache camstack-trainer \ + python -m camstack_train run --job /data/job.json --dataset /data/dataset.tar --out /data/out --cache /cache +``` + +Without Docker (`pip install ".[train,export]"` into an environment that has +torch), the same command runs as is. To only convert an export for training by +hand: + +```bash +python -m camstack_train dataset --export camstack-retrain.tar --out ./yolo-dataset +``` + +Tests (no torch, Ultralytics or OpenVINO needed): `pip install ".[test]" && pytest`. diff --git a/camstack_train/__init__.py b/camstack_train/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..59240f1bb79516264a516fc7e1bc625297af93ad --- /dev/null +++ b/camstack_train/__init__.py @@ -0,0 +1,9 @@ +"""CamStack trainer: fine-tune a root detector on a hub's annotated export. + +The hub never imports this package. It talks to it through files only: +``job.json`` + ``dataset.tar`` in, ``bundle.tar`` + ``result.json`` out, and +``CAMSTACK_EVENT`` lines on stdout. Ultralytics (AGPL-3.0) is imported only by +``train_det`` and the export steps, inside this image. +""" + +__version__ = "0.1.0" diff --git a/camstack_train/__main__.py b/camstack_train/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..09d05c367b3aff753671bff9a5c6597bfae6198f --- /dev/null +++ b/camstack_train/__main__.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +import sys + +from .cli import main + +sys.exit(main()) diff --git a/camstack_train/_steps.py b/camstack_train/_steps.py new file mode 100644 index 0000000000000000000000000000000000000000..9dfae1d2fac75603ac3161c8778ca95eec1f10c1 --- /dev/null +++ b/camstack_train/_steps.py @@ -0,0 +1,101 @@ +"""The heavy export steps, each run in its own child process. + +``python -m camstack_train._steps ''``. A crash inside +coremltools, NNCF or a converter takes down that child only; the parent +records the step as failed (``export.SubprocessToolchain``). The recipes mirror +``scripts/build-camstack-models.py`` / ``convert_lib.py`` in camstack/server +(the yolov9 recipe: static ONNX at opset 13, ``nms=False``, batch 1; CoreML +``half=True`` — FP16, D743; OpenVINO from the ONNX via ``ov.convert_model``, +never the Ultralytics OV exporter), and are not imported from them. +""" +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path + +#: Same bound as convert_lib / the inference pool (CAMSTACK_OV_COMPILATION_THREADS). +OV_COMPILATION_THREADS = 4 +ONNX_OPSET = 13 + + +def _move(src: Path, dest: Path) -> None: + if dest.exists(): + shutil.rmtree(dest) if dest.is_dir() else dest.unlink() + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(src), str(dest)) + + +def step_onnx(pt: str, out: str, imgsz: int) -> None: + from ultralytics import YOLO + + path = YOLO(pt).export( + format="onnx", imgsz=imgsz, dynamic=False, simplify=True, + opset=ONNX_OPSET, nms=False, batch=1, half=False, + ) + _move(Path(path), Path(out)) + + +def step_coreml(pt: str, out: str, imgsz: int) -> None: + from ultralytics import YOLO + + path = YOLO(pt).export(format="coreml", imgsz=imgsz, dynamic=False, batch=1, half=True, nms=False) + _move(Path(path), Path(out)) + + +def _validate_ir(xml: str) -> None: + import openvino as ov + + core = ov.Core() + compiled = core.compile_model(core.read_model(xml), "CPU", {"COMPILATION_NUM_THREADS": OV_COMPILATION_THREADS}) + for port in list(compiled.inputs) + list(compiled.outputs): + if not port.get_any_name(): + raise ValueError("the IR has an unnamed tensor") + + +def step_openvino(onnx: str, xml: str, fp16: bool) -> None: + import openvino as ov + + Path(xml).parent.mkdir(parents=True, exist_ok=True) + ov.save_model(ov.convert_model(onnx), xml, compress_to_fp16=fp16) + _validate_ir(xml) + + +def step_openvino_int8(onnx: str, xml: str, calib: list[str], imgsz: int) -> None: + import nncf + import numpy as np + import openvino as ov + + from ._vendor import ensure_pipeline_on_path + + ensure_pipeline_on_path() + import inference_pool # the pool's own decode + letterbox (grey 114, /255) + + def preprocess(path: str): + arr, _scale, _pad = inference_pool.letterbox(inference_pool.decode_jpeg(Path(path).read_bytes()), imgsz) + return arr.transpose(2, 0, 1)[np.newaxis].astype(np.float32) + + dataset = nncf.Dataset(calib, preprocess) + quantized = nncf.quantize(ov.convert_model(onnx), dataset, subset_size=len(calib)) + Path(xml).parent.mkdir(parents=True, exist_ok=True) + ov.save_model(quantized, xml) + _validate_ir(xml) + + +STEPS = { + "onnx": lambda a: step_onnx(a["pt"], a["out"], a["imgsz"]), + "coreml": lambda a: step_coreml(a["pt"], a["out"], a["imgsz"]), + "openvino": lambda a: step_openvino(a["onnx"], a["xml"], a["fp16"]), + "openvino-int8": lambda a: step_openvino_int8(a["onnx"], a["xml"], a["calib"], a["imgsz"]), +} + + +def main(argv: list[str]) -> int: + step, payload = argv[0], json.loads(argv[1]) + STEPS[step](payload) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/camstack_train/_vendor/__init__.py b/camstack_train/_vendor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b75e3460bd61f11043c97dde3d5fb6a39636869 --- /dev/null +++ b/camstack_train/_vendor/__init__.py @@ -0,0 +1,31 @@ +"""Byte-identical copies of CamStack code the trainer reuses, never edited here. + +- ``camstack_eval/`` is ``scripts/eval/camstack_eval/`` (the P5 evaluation + harness): the letterbox + decode + metrics + D739 floor rule the hub's own + measurements use. +- ``pipeline_python/`` is ``packages/addon-pipeline/python/`` (the inference + pool and its postprocessors, tests excluded): the harness imports the pool's + preprocess and decode from it, so the trainer evaluates exactly as a node runs. + +``scripts/check-trainer-vendor.ts`` in camstack/server fails when either copy +drifts from its source. Change the source, then re-copy. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +#: The folder holding ``inference_pool.py`` and ``postprocessors/``. +PIPELINE_PYTHON = Path(__file__).resolve().parent / "pipeline_python" + + +def ensure_pipeline_on_path() -> None: + """Put the vendored pool first on ``sys.path``. + + The harness's ``pipeline._pipeline()`` imports ``inference_pool`` and + ``postprocessors`` as top-level modules from the repository path; with this + folder already on the path those imports resolve to the vendored copy. + """ + entry = str(PIPELINE_PYTHON) + if entry not in sys.path: + sys.path.insert(0, entry) diff --git a/camstack_train/_vendor/camstack_eval/__init__.py b/camstack_train/_vendor/camstack_eval/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..23d89a52a86c95c6314916a25b30c698a299f5b7 --- /dev/null +++ b/camstack_train/_vendor/camstack_eval/__init__.py @@ -0,0 +1,6 @@ +"""CamStack detector evaluation harness (cloud-training P5). + +Pipeline-identical preprocess and decode for YOLO ONNX detectors, P/R/AP50 per +macro class, camera, day/night and small subjects, a baseline vs candidate +comparison on a holdout, and the D739 floor rule. See `scripts/eval/evaluate.py`. +""" diff --git a/camstack_train/_vendor/camstack_eval/bootstrap.py b/camstack_train/_vendor/camstack_eval/bootstrap.py new file mode 100644 index 0000000000000000000000000000000000000000..372576ea56d0f156c6b911b1f3e8a37e7f685f8d --- /dev/null +++ b/camstack_train/_vendor/camstack_eval/bootstrap.py @@ -0,0 +1,107 @@ +"""Paired bootstrap of an AP difference between two models on one set. + +Both models are scored on the SAME resampled frames (frames drawn with +replacement, ``rounds`` times), so what varies is the set, not the pairing: +the 95 % interval of ``AP(candidate) - AP(baseline)`` is the noise band a +"no regression" verdict is read against. Matching is per frame, so it is done +once (`metrics.frame_matches`) and a resample only re-weights each frame's +detections by how often the frame was drawn — the same AP as concatenating the +drawn frames (`metrics.ap_of_sample`), computed for every round at once. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional, Sequence + +import numpy as np + +from .metrics import COCO_IOUS, FrameMatches + + +@dataclass(frozen=True) +class _Flat: + """One model's matches of one class at one IoU, score-sorted.""" + + frame: np.ndarray # frame index per detection, highest score first + tp: np.ndarray # bool per detection + positives: np.ndarray # ground-truth count per frame + + +def _flatten(matches: Sequence[FrameMatches]) -> _Flat: + scores, tps, frames = [], [], [] + for i, m in enumerate(matches): + scores.extend(m.scores) + tps.extend(m.true_positive) + frames.extend([i] * len(m.scores)) + order = np.argsort(-np.asarray(scores, dtype=np.float64), kind="stable") + return _Flat( + frame=np.asarray(frames, dtype=np.int64)[order], + tp=np.asarray(tps, dtype=bool)[order], + positives=np.asarray([m.positives for m in matches], dtype=np.float64), + ) + + +def weighted_ap(flat: _Flat, weights: np.ndarray) -> np.ndarray: + """AP per round; ``weights`` is ``[rounds, frames]`` (draw counts).""" + total = weights @ flat.positives + if flat.frame.size == 0: + return np.where(total > 0, 0.0, np.nan) + w = weights[:, flat.frame] + ctp = np.cumsum(w * flat.tp, axis=1) + cfp = np.cumsum(w * ~flat.tp, axis=1) + denom = ctp + cfp + precision = np.divide(ctp, denom, out=np.zeros_like(ctp), where=denom > 0) + envelope = np.maximum.accumulate(precision[:, ::-1], axis=1)[:, ::-1] + gained = (w * flat.tp * envelope).sum(axis=1) + return np.divide(gained, total, out=np.full_like(total, np.nan), where=total > 0) + + +@dataclass(frozen=True) +class Delta: + """``candidate - baseline`` on the full set and its 95 % bootstrap interval.""" + + value: float + low: float + high: float + + @property + def significant_drop(self) -> bool: + return self.high < 0 + + @property + def significant_gain(self) -> bool: + return self.low > 0 + + def as_dict(self) -> dict: + return {"delta": round(self.value, 4), "ci95": [round(self.low, 4), round(self.high, 4)]} + + +def draw_weights(frames: int, rounds: int, seed: int = 0) -> np.ndarray: + """``[rounds + 1, frames]`` draw counts; row 0 is the full set (all ones).""" + rng = np.random.default_rng(seed) + draws = rng.integers(0, frames, size=(rounds, frames)) + counts = np.zeros((rounds, frames), dtype=np.float64) + np.add.at(counts, (np.arange(rounds)[:, None], draws), 1.0) + return np.vstack([np.ones((1, frames)), counts]) + + +def ap_rounds(matches_by_iou: dict[float, Sequence[FrameMatches]], weights: np.ndarray, + ious: Sequence[float]) -> np.ndarray: + """Mean AP over ``ious`` per round.""" + return np.nanmean( + np.stack([weighted_ap(_flatten(matches_by_iou[t]), weights) for t in ious]), axis=0 + ) + + +def paired_delta(baseline: np.ndarray, candidate: np.ndarray) -> Optional[Delta]: + """From per-round values (row 0 = full set) of two models.""" + diff = candidate - baseline + if np.isnan(diff[0]): + return None + rounds = diff[1:][~np.isnan(diff[1:])] + low, high = (np.percentile(rounds, [2.5, 97.5]) if rounds.size else (np.nan, np.nan)) + return Delta(float(diff[0]), float(low), float(high)) + + +AP50 = (0.5,) +AP50_95 = COCO_IOUS diff --git a/camstack_train/_vendor/camstack_eval/classes.py b/camstack_train/_vendor/camstack_eval/classes.py new file mode 100644 index 0000000000000000000000000000000000000000..5182b64386dad8b3e7381a41e14b0996d3867442 --- /dev/null +++ b/camstack_train/_vendor/camstack_eval/classes.py @@ -0,0 +1,40 @@ +"""COCO → CamStack macro classes, as the pipeline maps them +(`packages/types/src/catalogs/coco-classmap.ts` `COCO_TO_MACRO`). The floors +and the comparison are per macro class, because that is what the pipeline +gates on. `package` (COCO luggage proxies) is mapped but not evaluated by +default: it was too rare to calibrate (D739 keeps it at 0.50).""" +from __future__ import annotations + +from typing import Optional + +from .metrics import Detection + +COCO_TO_MACRO: dict[str, str] = { + "person": "person", + **{k: "vehicle" for k in ("bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat")}, + **{ + k: "animal" + for k in ("bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe") + }, + **{k: "package" for k in ("suitcase", "backpack", "handbag")}, +} + +#: Macro classes a root detector is evaluated on. +DETECTOR_CLASSES: tuple[str, ...] = ("person", "vehicle", "animal") + + +def macro_of(label: str) -> Optional[str]: + """A label's macro class; a label that already IS a macro maps to itself.""" + if label in DETECTOR_CLASSES or label == "package": + return label + return COCO_TO_MACRO.get(label) + + +def to_macro(detections: list[Detection]) -> list[Detection]: + """Map detections to macro classes; a class the pipeline drops is dropped.""" + out = [] + for d in detections: + macro = macro_of(d.cls) + if macro is not None: + out.append(Detection(macro, d.score, d.bbox)) + return out diff --git a/camstack_train/_vendor/camstack_eval/compare.py b/camstack_train/_vendor/camstack_eval/compare.py new file mode 100644 index 0000000000000000000000000000000000000000..ad1944db635ae06c2f3f8ebc70c92ea64fa9093c --- /dev/null +++ b/camstack_train/_vendor/camstack_eval/compare.py @@ -0,0 +1,102 @@ +"""Baseline vs candidate on one holdout. + +Both models are run on the same frames. The candidate's floors come from the +D739 rule against the baseline's shipped floors (`floors.propose_floor`), and +the candidate is reported AT those floors — "same false-alert rate, what more +does it find". Per-camera recall deltas are what a promotion gate reads +(analysis §6: no camera may lose more than 0.01 of person recall). +""" +from __future__ import annotations + +from typing import Callable, Iterable, Optional, Sequence + +from .classes import DETECTOR_CLASSES, to_macro +from .dataset import EvalFrame +from .floors import FloorPolicy, ScoredDetection, propose_floor +from .metrics import ClassResult, Detection, FrameTruth, average_precision, evaluate_class, scored_detections + +Detector = Callable[[bytes], Sequence[Detection]] + + +def select_holdout( + frames: Iterable[EvalFrame], + cameras: Optional[Sequence[int]] = None, + since: Optional[int] = None, +) -> list[EvalFrame]: + """Holdout by whole cameras and/or by capture time — never by random frame.""" + wanted = set(cameras) if cameras else None + out = [] + for f in frames: + if wanted is not None and f.truth.camera not in wanted: + continue + if since is not None and (f.captured_at is None or f.captured_at < since): + continue + out.append(f) + return out + + +def _run(frames: Sequence[EvalFrame], detector: Detector) -> list[tuple[FrameTruth, list[Detection]]]: + return [(f.truth, to_macro(list(detector(f.read())))) for f in frames] + + +def _summary(result: ClassResult, ap: Optional[float]) -> dict: + return { + "floor": result.floor, + "tp": result.tp, + "fp": result.fp, + "fn": result.fn, + "precision": result.precision, + "recall": result.recall, + "ap50": ap, + "recallByCamera": {str(k): v for k, v in sorted(result.recall_by_camera.items())}, + "recallByLight": dict(sorted(result.recall_by_light.items())), + "fpByCamera": {str(k): v for k, v in sorted(result.fp_by_camera.items())}, + "smallFound": result.small_found, + "smallTotal": result.small_total, + } + + +def compare_models( + frames: Sequence[EvalFrame], + baseline: Detector, + candidate: Detector, + baseline_floors: dict[str, float], + candidate_floors: Optional[dict[str, float]] = None, + policy: FloorPolicy = FloorPolicy(), + classes: Sequence[str] = DETECTOR_CLASSES, +) -> dict: + base_runs = _run(frames, baseline) + cand_runs = _run(frames, candidate) + report: dict = {"frames": len(frames), "classes": {}} + for cls in classes: + base_floor = baseline_floors[cls] + base_scored, positives = scored_detections(cls, base_runs) + cand_scored, _ = scored_detections(cls, cand_runs) + proposed = propose_floor( + [ScoredDetection(d.score, d.true_positive) for d in base_scored], + base_floor, + [ScoredDetection(d.score, d.true_positive) for d in cand_scored], + positives, + policy, + ) + cand_floor = (candidate_floors or {}).get(cls, proposed.value) + base = evaluate_class(cls, base_runs, base_floor) + cand = evaluate_class(cls, cand_runs, cand_floor) + deltas = {} + for cam, recall in cand.recall_by_camera.items(): + before = base.recall_by_camera.get(cam) + if recall is not None and before is not None: + deltas[str(cam)] = round(recall - before, 4) + report["classes"][cls] = { + "positives": positives, + "baseline": _summary(base, average_precision(cls, base_runs)), + "candidate": _summary(cand, average_precision(cls, cand_runs)), + "proposedFloor": { + "value": proposed.value, + "reason": proposed.reason, + "fpBudget": proposed.fp_budget, + "fitted": proposed.fitted, + }, + "recallDeltaByCamera": dict(sorted(deltas.items())), + } + return report diff --git a/camstack_train/_vendor/camstack_eval/dataset.py b/camstack_train/_vendor/camstack_eval/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..adaa06958bef26645d129d9bed41dade9a7fcdeb --- /dev/null +++ b/camstack_train/_vendor/camstack_eval/dataset.py @@ -0,0 +1,181 @@ +"""The hub's annotated export (`annotations.json`, schema +`camstack.retrain-export/v2`) as an evaluation set. + +Only frames the operator marked ``complete`` ("all subjects boxed") are ground +truth by default: on any other frame an unboxed person would be scored as a +false positive. ``subject`` annotations of a detector macro class are the +positives; ``model_error`` boxes mark a region the model got wrong and are not +positives (a detection there is a false positive, as it should be). + +Works on the tar itself or on an extracted directory. +""" +from __future__ import annotations + +import json +import tarfile +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Optional + +from .classes import DETECTOR_CLASSES +from .metrics import FrameTruth, GroundTruthBox + +MANIFEST = "annotations.json" + + +@dataclass(frozen=True) +class EvalFrame: + truth: FrameTruth + path: str + captured_at: Optional[int] + read: Callable[[], bytes] + + +def _night(entry: dict) -> Optional[bool]: + lighting = entry.get("lighting") + return None if not isinstance(lighting, dict) else bool(lighting.get("night")) + + +def _truth(entry: dict, classes: tuple[str, ...]) -> FrameTruth: + boxes = [] + for a in entry.get("annotations", []): + if a.get("kind") != "subject" or a.get("macroClass") not in classes: + continue + b = a["bbox"] + boxes.append(GroundTruthBox(a["macroClass"], (b["x"], b["y"], b["w"], b["h"]))) + return FrameTruth( + frame_id=entry["frameId"], + camera=int(entry["deviceId"]), + night=_night(entry), + boxes=tuple(boxes), + ) + + +def load_export( + source: str, + complete_only: bool = True, + classes: tuple[str, ...] = DETECTOR_CLASSES, +) -> list[EvalFrame]: + root = Path(source) + if root.is_dir(): + manifest = json.loads((root / MANIFEST).read_text("utf-8")) + + def reader(path: str) -> Callable[[], bytes]: + return lambda: (root / path).read_bytes() + else: + with tarfile.open(root) as tar: + member = tar.extractfile(MANIFEST) + if member is None: + raise ValueError(f"{source}: no {MANIFEST}") + manifest = json.loads(member.read().decode("utf-8")) + + def reader(path: str) -> Callable[[], bytes]: + def read() -> bytes: + with tarfile.open(root) as tar: + f = tar.extractfile(path) + if f is None: + raise FileNotFoundError(path) + return f.read() + + return read + + out = [] + for entry in manifest.get("frames", []): + if complete_only and entry.get("complete") is not True: + continue + out.append( + EvalFrame( + truth=_truth(entry, classes), + path=entry["path"], + captured_at=entry.get("capturedAt"), + read=reader(entry["path"]), + ) + ) + return out + + +def _file_reader(path: Path) -> Callable[[], bytes]: + return lambda: path.read_bytes() + + +def load_coco( + annotations: str, + images_dir: str, + skip: int = 0, + limit: Optional[int] = None, + classes: tuple[str, ...] = DETECTOR_CLASSES, +) -> list[EvalFrame]: + """COCO ``instances_*.json`` as an evaluation set, in macro classes. + + Images are taken in FILE-NAME order — the order the INT8 calibration reads + the same folder (`convert_lib.calibration_images`) — after the first + ``skip``, so ``skip`` = the calibration subset size keeps every calibration + image out of the holdout. Every image counts, including those with no + object of a macro class: a detection there is a false positive. A crowd + box (``iscrowd``) is "don't care", as in COCO's own evaluation. + """ + from .classes import macro_of + + data = json.loads(Path(annotations).read_text("utf-8")) + names = {c["id"]: c["name"] for c in data["categories"]} + by_image: dict[int, list[dict]] = {} + for a in data["annotations"]: + by_image.setdefault(a["image_id"], []).append(a) + images = sorted(data["images"], key=lambda im: im["file_name"])[skip:] + if limit is not None: + images = images[:limit] + root = Path(images_dir) + out = [] + for im in images: + w, h = float(im["width"]), float(im["height"]) + boxes, dont_care = [], [] + for a in by_image.get(im["id"], []): + macro = macro_of(names[a["category_id"]]) + if macro not in classes: + continue + x, y, bw, bh = a["bbox"] + gt = GroundTruthBox(macro, (x / w, y / h, bw / w, bh / h)) + (dont_care if a.get("iscrowd") else boxes).append(gt) + out.append( + EvalFrame( + truth=FrameTruth( + frame_id=im["file_name"], camera=0, night=None, + boxes=tuple(boxes), dont_care=tuple(dont_care), + ), + path=im["file_name"], + captured_at=None, + read=_file_reader(root / im["file_name"]), + ) + ) + return out + + +def load_pseudo_gt(source: str, frames_dir: Optional[str] = None) -> list[EvalFrame]: + """A pseudo ground truth (schema ``camstack.pseudo-gt/v1``): per frame the + positives and the "don't care" boxes of `pseudo_gt.pseudo_ground_truth`, + normalised ``(x, y, w, h)``. Frames are read from ``frames_dir`` (default: + the folder holding the JSON).""" + path = Path(source) + data = json.loads(path.read_text("utf-8")) + if data.get("schema") != "camstack.pseudo-gt/v1": + raise ValueError(f"{source}: not a camstack.pseudo-gt/v1 file") + root = Path(frames_dir) if frames_dir else path.parent + + def boxes(rows: list[dict]) -> tuple[GroundTruthBox, ...]: + return tuple(GroundTruthBox(r["cls"], tuple(r["bbox"])) for r in rows) + + return [ + EvalFrame( + truth=FrameTruth( + frame_id=f["file"], + camera=int(f.get("deviceId", 0)), + night=f.get("night"), + boxes=boxes(f["boxes"]), + dont_care=boxes(f.get("dontCare", [])), + ), + path=f["file"], + captured_at=f.get("capturedAt"), + read=_file_reader(root / f["file"]), + ) + for f in data["frames"] + ] diff --git a/camstack_train/_vendor/camstack_eval/floors.py b/camstack_train/_vendor/camstack_eval/floors.py new file mode 100644 index 0000000000000000000000000000000000000000..d42f3100da9198c7647ed6ea6865583610b27d41 --- /dev/null +++ b/camstack_train/_vendor/camstack_eval/floors.py @@ -0,0 +1,79 @@ +"""The D739 floor rule (docs/decisions/adr-0739-*.md). + +A model's floor for a class is the LOWEST confidence at which it raises no more +false positives on the evaluation set than the baseline does at the baseline's +shipped floor — so switching model keeps the false-alert rate and changes only +what is found. On a small or fine-tuned set two guards apply: + +- **support**: a class with fewer than ``min_support`` ground-truth positives + inherits the baseline floor (a floor fitted on a handful of boxes is noise); +- **clamp**: the floor stays within ``±clamp`` of the baseline floor (single + digit false-positive counts carry ±0.05–0.10 of noise). + +Floors are on a ``grid_step`` grid and never below ``minimum``, as in the +2026-10-07 calibration. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal, Optional, Sequence + +FloorReason = Literal["fp-budget", "low-support", "clamped"] + + +@dataclass(frozen=True) +class ScoredDetection: + score: float + true_positive: bool + + +@dataclass(frozen=True) +class FloorPolicy: + min_support: int = 20 + clamp: float = 0.15 + grid_step: float = 0.05 + minimum: float = 0.25 + maximum: float = 0.95 + + +@dataclass(frozen=True) +class ProposedFloor: + value: float + reason: FloorReason + fp_budget: int + #: The unclamped fit, ``None`` when no grid floor meets the budget. + fitted: Optional[float] + + +def false_positives_at(detections: Sequence[ScoredDetection], floor: float) -> int: + return sum(1 for d in detections if not d.true_positive and d.score >= floor - 1e-9) + + +def _grid(policy: FloorPolicy) -> list[float]: + steps = int(round((policy.maximum - policy.minimum) / policy.grid_step)) + return [round(policy.minimum + i * policy.grid_step, 4) for i in range(steps + 1)] + + +def propose_floor( + baseline: Sequence[ScoredDetection], + baseline_floor: float, + candidate: Sequence[ScoredDetection], + positives: int, + policy: FloorPolicy = FloorPolicy(), +) -> ProposedFloor: + budget = false_positives_at(baseline, baseline_floor) + if positives < policy.min_support: + return ProposedFloor(baseline_floor, "low-support", budget, None) + fitted = next( + (t for t in _grid(policy) if false_positives_at(candidate, t) <= budget), + None, + ) + low = round(max(policy.minimum, baseline_floor - policy.clamp), 4) + high = round(baseline_floor + policy.clamp, 4) + if fitted is None: + return ProposedFloor(high, "clamped", budget, None) + if fitted < low: + return ProposedFloor(low, "clamped", budget, fitted) + if fitted > high: + return ProposedFloor(high, "clamped", budget, fitted) + return ProposedFloor(fitted, "fp-budget", budget, fitted) diff --git a/camstack_train/_vendor/camstack_eval/metrics.py b/camstack_train/_vendor/camstack_eval/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..3c92863d7a68fa819d945f2df14d12f4e9b44977 --- /dev/null +++ b/camstack_train/_vendor/camstack_eval/metrics.py @@ -0,0 +1,273 @@ +"""Detection metrics against a ground truth, the way the 2026-10-07 detector +comparison measured them (docs/benchmarks/2026-10-07-detector-comparison.md). + +Boxes are normalised ``(x, y, w, h)`` against the full frame. A detection is a +true positive when it overlaps an unmatched ground-truth box of the same macro +class at IoU >= 0.5 (greedy, highest score first); a detection on a +"don't care" box is ignored; everything else is a false positive. Greedy in +score order makes the matching threshold-consistent: raising the floor only +truncates the list, so one pass serves every floor and AP50. + +"Small" follows the comparison: a box under 100 px tall in a 960x540 frame, +i.e. under 100/540 of the frame height. + +AP50-95 is the COCO average of AP over the IoU thresholds 0.50, 0.55 … 0.95, +each matched the same greedy way (a "don't care" box is still ignored at IoU +0.5). It is computed per macro class, at the pool floor, so it is lower than an +Ultralytics `val` mAP (floor 0.001, 80 classes): compare it only with itself. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Iterable, Optional, Sequence + +Box = tuple[float, float, float, float] + +MATCH_IOU = 0.5 +SMALL_HEIGHT_FRACTION = 100 / 540 +#: COCO's AP50-95 thresholds. +COCO_IOUS: tuple[float, ...] = tuple(round(0.5 + 0.05 * i, 2) for i in range(10)) + + +@dataclass(frozen=True) +class GroundTruthBox: + cls: str + bbox: Box + + +@dataclass(frozen=True) +class Detection: + cls: str + score: float + bbox: Box + + +@dataclass(frozen=True) +class FrameTruth: + frame_id: str + camera: int + #: ``True`` night/IR, ``False`` day, ``None`` unknown. + night: Optional[bool] + boxes: tuple[GroundTruthBox, ...] + dont_care: tuple[GroundTruthBox, ...] = () + + +@dataclass(frozen=True) +class MatchedDetection: + """One detection of the class, after matching: what floors and AP use.""" + + score: float + true_positive: bool + + +@dataclass +class ClassResult: + cls: str + floor: float + tp: int = 0 + fp: int = 0 + fn: int = 0 + small_found: int = 0 + small_total: int = 0 + recall_by_camera: dict = field(default_factory=dict) + recall_by_light: dict = field(default_factory=dict) + fp_by_camera: dict = field(default_factory=dict) + + @property + def precision(self) -> Optional[float]: + return None if self.tp + self.fp == 0 else self.tp / (self.tp + self.fp) + + @property + def recall(self) -> Optional[float]: + return None if self.tp + self.fn == 0 else self.tp / (self.tp + self.fn) + + +def iou(a: Box, b: Box) -> float: + ax2, ay2 = a[0] + a[2], a[1] + a[3] + bx2, by2 = b[0] + b[2], b[1] + b[3] + iw = max(0.0, min(ax2, bx2) - max(a[0], b[0])) + ih = max(0.0, min(ay2, by2) - max(a[1], b[1])) + inter = iw * ih + union = a[2] * a[3] + b[2] * b[3] - inter + return inter / union if union > 0 else 0.0 + + +def is_small(bbox: Box) -> bool: + return bbox[3] < SMALL_HEIGHT_FRACTION + + +def light_bucket(night: Optional[bool]) -> str: + return "unknown" if night is None else ("night" if night else "day") + + +@dataclass(frozen=True) +class _FrameMatch: + #: (score, verdict) per detection of the class, highest score first; + #: verdict is "tp", "fp" or "ignored". + detections: tuple[tuple[float, str], ...] + #: per ground-truth box: the score that matched it, or None. + gt_scores: tuple[Optional[float], ...] + + +def _match_frame( + cls: str, truth: FrameTruth, detections: Sequence[Detection], match_iou: float = MATCH_IOU +) -> _FrameMatch: + gts = [g for g in truth.boxes if g.cls == cls] + cares = [g for g in truth.dont_care if g.cls == cls] + gt_scores: list[Optional[float]] = [None] * len(gts) + verdicts: list[tuple[float, str]] = [] + for det in sorted((d for d in detections if d.cls == cls), key=lambda d: -d.score): + best, best_iou = -1, match_iou + for i, g in enumerate(gts): + if gt_scores[i] is not None: + continue + overlap = iou(det.bbox, g.bbox) + if overlap >= best_iou: + best, best_iou = i, overlap + if best >= 0: + gt_scores[best] = det.score + verdicts.append((det.score, "tp")) + elif any(iou(det.bbox, c.bbox) >= MATCH_IOU for c in cares): + verdicts.append((det.score, "ignored")) + else: + verdicts.append((det.score, "fp")) + return _FrameMatch(tuple(verdicts), tuple(gt_scores)) + + +def _ratio(hit: int, total: int) -> Optional[float]: + return None if total == 0 else hit / total + + +def evaluate_class( + cls: str, + frames: Iterable[tuple[FrameTruth, Sequence[Detection]]], + floor: float, +) -> ClassResult: + result = ClassResult(cls=cls, floor=floor) + by_camera: dict[int, list[int]] = {} + by_light: dict[str, list[int]] = {} + for truth, detections in frames: + match = _match_frame(cls, truth, detections) + gts = [g for g in truth.boxes if g.cls == cls] + for score, verdict in match.detections: + if score < floor or verdict == "ignored": + continue + if verdict == "tp": + result.tp += 1 + else: + result.fp += 1 + result.fp_by_camera[truth.camera] = result.fp_by_camera.get(truth.camera, 0) + 1 + for g, score in zip(gts, match.gt_scores): + found = score is not None and score >= floor + if not found: + result.fn += 1 + cam = by_camera.setdefault(truth.camera, [0, 0]) + light = by_light.setdefault(light_bucket(truth.night), [0, 0]) + cam[0] += int(found) + cam[1] += 1 + light[0] += int(found) + light[1] += 1 + if is_small(g.bbox): + result.small_total += 1 + result.small_found += int(found) + result.recall_by_camera = {k: _ratio(*v) for k, v in by_camera.items()} + result.recall_by_light = {k: _ratio(*v) for k, v in by_light.items()} + return result + + +def scored_detections( + cls: str, + frames: Iterable[tuple[FrameTruth, Sequence[Detection]]], + match_iou: float = MATCH_IOU, +) -> tuple[list[MatchedDetection], int]: + """Every detection of the class with its verdict (ignored ones dropped), + and the number of ground-truth positives. Feeds AP50 and the floor rule.""" + out: list[MatchedDetection] = [] + positives = 0 + for truth, detections in frames: + match = _match_frame(cls, truth, detections, match_iou) + positives += sum(1 for g in truth.boxes if g.cls == cls) + out.extend(MatchedDetection(s, v == "tp") for s, v in match.detections if v != "ignored") + return out, positives + + +def ap_from_matches(scores: Sequence[float], true_positive: Sequence[bool], positives: int) -> Optional[float]: + """AP, all-point interpolated (VOC 2010+), from matched detections in any + order. ``None`` with no positives. Ties keep their given order.""" + if positives == 0: + return None + order = sorted(range(len(scores)), key=lambda i: -scores[i]) + tp = fp = 0 + precisions: list[float] = [] + hits: list[bool] = [] + for i in order: + hit = bool(true_positive[i]) + tp += int(hit) + fp += int(not hit) + precisions.append(tp / (tp + fp)) + hits.append(hit) + # Precision envelope: the best precision at this recall or any higher one. + best = 0.0 + for i in range(len(precisions) - 1, -1, -1): + best = max(best, precisions[i]) + precisions[i] = best + return sum(p for p, hit in zip(precisions, hits) if hit) / positives + + +def average_precision( + cls: str, + frames: Iterable[tuple[FrameTruth, Sequence[Detection]]], + match_iou: float = MATCH_IOU, +) -> Optional[float]: + """AP at IoU ``match_iou`` (0.5 by default), all-point interpolated (VOC 2010+). + ``None`` with no positives.""" + scored, positives = scored_detections(cls, frames, match_iou) + return ap_from_matches([d.score for d in scored], [d.true_positive for d in scored], positives) + + +def average_precision_coco( + cls: str, frames: Sequence[tuple[FrameTruth, Sequence[Detection]]] +) -> Optional[float]: + """AP50-95: the mean AP over ``COCO_IOUS``. ``None`` with no positives.""" + aps = [average_precision(cls, frames, t) for t in COCO_IOUS] + return None if aps[0] is None else sum(a for a in aps if a is not None) / len(aps) + + +@dataclass(frozen=True) +class FrameMatches: + """One frame's matched detections of one class at one IoU: what a + bootstrap re-uses, since matching is per frame.""" + + scores: tuple[float, ...] + true_positive: tuple[bool, ...] + positives: int + + +def frame_matches( + cls: str, frames: Sequence[tuple[FrameTruth, Sequence[Detection]]], match_iou: float +) -> list[FrameMatches]: + out = [] + for truth, detections in frames: + match = _match_frame(cls, truth, detections, match_iou) + kept = [(s, v == "tp") for s, v in match.detections if v != "ignored"] + out.append( + FrameMatches( + tuple(s for s, _ in kept), + tuple(t for _, t in kept), + sum(1 for g in truth.boxes if g.cls == cls), + ) + ) + return out + + +def ap_of_sample(matches: Sequence[FrameMatches], sample: Iterable[int]) -> Optional[float]: + """AP over the frames ``sample`` (indices, repeats allowed) of ``matches``.""" + scores: list[float] = [] + hits: list[bool] = [] + positives = 0 + for i in sample: + m = matches[i] + scores.extend(m.scores) + hits.extend(m.true_positive) + positives += m.positives + return ap_from_matches(scores, hits, positives) diff --git a/camstack_train/_vendor/camstack_eval/pipeline.py b/camstack_train/_vendor/camstack_eval/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..d7b090fcdea90b247e8de201ace73d62c8068a6e --- /dev/null +++ b/camstack_train/_vendor/camstack_eval/pipeline.py @@ -0,0 +1,170 @@ +"""Run a YOLO detector (ONNX or OpenVINO IR) exactly as the inference pool does. + +Nothing here re-implements the pipeline: the letterbox (grey 114 padding, +bilinear resize, /255) and the YOLO decode (sigmoid detection, confidence +floor, NMS) are IMPORTED from `packages/addon-pipeline/python`, so a change to +production preprocessing changes the evaluation with it. The pool floor (0.05) +and the NMS IoU (0.45) are the pool's. + +An OpenVINO IR is compiled as the pool compiles it: a static 3-channel input +gets the pool's own PrePostProcessor (uint8 NHWC in, the /255 folded into the +graph — `inference_pool._build_ov_ppp_model`), a dynamic one is fed the float +letterbox. The device is the CPU at FP32 (`INFERENCE_PRECISION_HINT=f32`), the +x86 CPU default; an ARM CPU defaults to FP16, which is not what any node runs. + +One difference stays and is stated: the live path first downscales a frame to +the model's content size on the Node side (sharp); here the PIL letterbox does +that resize itself. The comparison run of 2026-10-07 measured the same way. +""" +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Optional, Sequence + +import numpy as np + +from .metrics import Detection + +REPO_ROOT = Path(__file__).resolve().parents[3] +PIPELINE_PYTHON = REPO_ROOT / "packages" / "addon-pipeline" / "python" + +#: The inference pool's confidence floor and NMS IoU. +POOL_FLOOR = 0.05 +POOL_NMS_IOU = 0.45 + + +def _pipeline(): + if str(PIPELINE_PYTHON) not in sys.path: + sys.path.insert(0, str(PIPELINE_PYTHON)) + import inference_pool # noqa: E402 (path set above) + from postprocessors import POSTPROCESSORS # noqa: E402 + + return inference_pool, POSTPROCESSORS + + +class OnnxDetector: + """A detector on ONNX Runtime (CPU unless ``providers`` says otherwise).""" + + def __init__( + self, + model_path: str, + input_size: int, + labels: Optional[Sequence[str]] = None, + postprocessor: str = "yolo", + pool_floor: float = POOL_FLOOR, + nms_iou: float = POOL_NMS_IOU, + providers: Optional[Sequence[str]] = None, + ) -> None: + import onnxruntime as ort + + self._pool, postprocessors = _pipeline() + if postprocessor not in postprocessors: + raise ValueError(f"unknown postprocessor {postprocessor!r}") + self._postprocess = postprocessors[postprocessor] + from postprocessors.yolo import COCO_80 + + self.input_size = input_size + self._config = { + "inputSize": input_size, + "labels": list(labels) if labels is not None else list(COCO_80), + "confidence": pool_floor, + "nmsIouThreshold": nms_iou, + "postprocessor": postprocessor, + } + self._session = ort.InferenceSession( + model_path, providers=list(providers) if providers else ["CPUExecutionProvider"] + ) + self._input_name = self._session.get_inputs()[0].name + self._output_names = [o.name for o in self._session.get_outputs()] + + def detect(self, jpeg: bytes) -> list[Detection]: + img = self._pool.decode_jpeg(jpeg) + orig_w, orig_h = img.size + arr, scale, pad = self._pool.letterbox(img, self.input_size) + tensor = arr.transpose(2, 0, 1)[np.newaxis].astype(np.float32) + outputs = self._session.run(self._output_names, {self._input_name: tensor}) + predictions = dict(zip(self._output_names, outputs)) + result = self._postprocess(predictions, self._config, orig_w, orig_h, scale, pad) + return _to_detections(result, orig_w, orig_h) + + +def _to_detections(result: dict, orig_w: int, orig_h: int) -> list[Detection]: + out: list[Detection] = [] + for d in result.get("detections", []): + x1, y1, x2, y2 = d["bbox"] + out.append( + Detection( + str(d["class"]), + float(d["score"]), + (x1 / orig_w, y1 / orig_h, (x2 - x1) / orig_w, (y2 - y1) / orig_h), + ) + ) + return out + + +class OpenVinoDetector: + """A detector on an OpenVINO IR, on the CPU at FP32 (see the module doc).""" + + def __init__( + self, + model_path: str, + input_size: int, + labels: Optional[Sequence[str]] = None, + postprocessor: str = "yolo", + pool_floor: float = POOL_FLOOR, + nms_iou: float = POOL_NMS_IOU, + device: str = "CPU", + precision: str = "f32", + ) -> None: + import openvino as ov + + self._pool, postprocessors = _pipeline() + if postprocessor not in postprocessors: + raise ValueError(f"unknown postprocessor {postprocessor!r}") + self._postprocess = postprocessors[postprocessor] + from postprocessors.yolo import COCO_80 + + self.input_size = input_size + self._config = { + "inputSize": input_size, + "labels": list(labels) if labels is not None else list(COCO_80), + "confidence": pool_floor, + "nmsIouThreshold": nms_iou, + "postprocessor": postprocessor, + "preprocessMode": "letterbox", + } + core = ov.Core() + model, self._ppp = self._pool._build_ov_ppp_model(core, model_path, self._config) + properties = {"INFERENCE_PRECISION_HINT": precision, "PERFORMANCE_HINT": "LATENCY"} + self._compiled = core.compile_model(model if model is not None else model_path, device, properties) + self._request = self._compiled.create_infer_request() + self._output_names = [o.get_any_name() for o in self._compiled.outputs] + + @property + def folded(self) -> bool: + """True when the pool's PrePostProcessor was attached (static input).""" + return self._ppp is not None + + def detect(self, jpeg: bytes) -> list[Detection]: + img = self._pool.decode_jpeg(jpeg) + orig_w, orig_h = img.size + if self._ppp is not None: + tensor, scale, pad = self._pool._preprocess_ov_uint8(img, self._ppp) + else: + arr, scale, pad = self._pool.letterbox(img, self.input_size) + tensor = arr.transpose(2, 0, 1)[np.newaxis].astype(np.float32) + self._request.infer({0: tensor}) + predictions = { + name: self._request.get_output_tensor(i).data.copy() + for i, name in enumerate(self._output_names) + } + result = self._postprocess(predictions, self._config, orig_w, orig_h, scale, pad) + return _to_detections(result, orig_w, orig_h) + + +def make_detector(model_path: str, input_size: int): + """An ``OpenVinoDetector`` for an ``.xml`` IR, an ``OnnxDetector`` otherwise.""" + if model_path.endswith(".xml"): + return OpenVinoDetector(model_path, input_size) + return OnnxDetector(model_path, input_size) diff --git a/camstack_train/_vendor/camstack_eval/pseudo_gt.py b/camstack_train/_vendor/camstack_eval/pseudo_gt.py new file mode 100644 index 0000000000000000000000000000000000000000..019bd499838406811fd3a1b76fbecb0e2c95b8fc --- /dev/null +++ b/camstack_train/_vendor/camstack_eval/pseudo_gt.py @@ -0,0 +1,68 @@ +"""Pseudo ground truth from two teacher detectors — the 2026-10-07 method. + +"Two large reference detectors (YOLO26-X at 1280 px and RT-DETR-X at 640 px) +are run on every frame; an object counts when both agree (or one is very +confident). Uncertain boxes are don't care." The original scripts were lost +with a session scratchpad; the thresholds below are the documented SHAPE of +the rule with defaults chosen here, not the exact values of that run. +Detections must already be in macro classes (`classes.to_macro`). +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Sequence + +from .metrics import MATCH_IOU, Detection, GroundTruthBox, iou + + +@dataclass(frozen=True) +class PseudoGtPolicy: + #: Both teachers at or above this, same class, IoU >= 0.5 ⇒ positive. + agree_score: float = 0.3 + #: One teacher alone at or above this ⇒ positive. + solo_score: float = 0.8 + #: A box a teacher holds at or above this that is not a positive ⇒ don't care. + care_score: float = 0.3 + + +def pseudo_ground_truth( + teacher_a: Sequence[Detection], + teacher_b: Sequence[Detection], + policy: PseudoGtPolicy = PseudoGtPolicy(), +) -> tuple[list[GroundTruthBox], list[GroundTruthBox]]: + positives: list[GroundTruthBox] = [] + used_b: set[int] = set() + leftovers: list[Detection] = [] + for a in sorted(teacher_a, key=lambda d: -d.score): + partner = -1 + if a.score >= policy.agree_score: + for j, b in enumerate(teacher_b): + if j in used_b or b.cls != a.cls or b.score < policy.agree_score: + continue + if iou(a.bbox, b.bbox) >= MATCH_IOU: + partner = j + break + if partner >= 0: + used_b.add(partner) + positives.append(GroundTruthBox(a.cls, a.bbox)) + elif a.score >= policy.solo_score: + positives.append(GroundTruthBox(a.cls, a.bbox)) + else: + leftovers.append(a) + for j, b in enumerate(teacher_b): + if j in used_b: + continue + duplicate = any(p.cls == b.cls and iou(p.bbox, b.bbox) >= MATCH_IOU for p in positives) + if duplicate: + continue + if b.score >= policy.solo_score: + positives.append(GroundTruthBox(b.cls, b.bbox)) + else: + leftovers.append(b) + dont_care = [ + GroundTruthBox(d.cls, d.bbox) + for d in leftovers + if d.score >= policy.care_score + and not any(p.cls == d.cls and iou(p.bbox, d.bbox) >= MATCH_IOU for p in positives) + ] + return positives, dont_care diff --git a/camstack_train/_vendor/pipeline_python/inference_pool.py b/camstack_train/_vendor/pipeline_python/inference_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..1194a043d4a60fb6bf12ade4a3f764d3ced79c5a --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/inference_pool.py @@ -0,0 +1,3464 @@ +#!/usr/bin/env python3 +"""Async inference pool — request_id multiplexed, per-runtime concurrency. + +Per-runtime ML backends (coreml / openvino / onnx): + - asyncio main loop reads requests from stdin; inference is dispatched + onto a runtime-specific executor so the reader never blocks. + - Each request carries a 32-bit id; responses are tagged with the same + id so the Node side can keep N requests in flight concurrently. + - Runtime executors: + CoreML → ThreadPoolExecutor(1) — ANE is single-context; one + Python thread is enough, and avoids GIL thrashing. + OpenVINO → ThreadPoolExecutor(OPTIMAL_NUMBER_OF_INFER_REQUESTS), + each worker checking out an InferRequest from a fixed + per-model pool (created once, post-compile), with the model + compiled on ONE explicit device under the LATENCY hint (no + AUTO — its CPU->GPU handover invalidated in-flight requests). + ONNX → ThreadPoolExecutor(N) — N independent InferenceSessions + where N = concurrency setting; each session pinned to + its own worker thread. + - Optional raw-frame path skips JPEG decode when the caller already + has a decoded RGB/BGR/GRAY buffer (e.g. the stream-broker decoder + cap output). + +Startup protocol (Node → Python, v2): + 1. [4B total_len][4B req_id=0][1B msg_type=0x00][JSON config] + config = {"runtime": ..., "models": [...], "concurrency": N?} + 2. Python responds with [4B total_len][4B req_id=0][JSON ready status]. + +Runtime protocol: + Request: [4B total_len][4B req_id][1B msg_type][payload] + Response: [4B total_len][4B req_id][JSON payload] + + msg_type: + 0x00 — command payload = JSON + 0x01 — infer_jpeg payload = [1B model_idx][JPEG bytes] + 0x02 — infer_raw payload = [1B model_idx][4B width][4B height] + [1B fmt 0=RGB,1=BGR,2=GRAY][pixels] + +Wire-protocol versioning (D350): the startup config may declare +`protocolVersion`; the effective wire version is `min(PROTOCOL_VERSION, +declared)` and the ready reply reports this worker's own PROTOCOL_VERSION so +the Node side can negotiate the same minimum. At v2, every SHEDDABLE request +(infer_jpeg / infer_raw / infer_batch / infer_cached) prefixes its payload +with the request's ABSOLUTE deadline — [8B epoch-ms uint64 LE] — and this +worker checks it immediately before predict, answering +`{"dropped": true, "shedReason": "deadline-expired"}` instead of executing +work whose caller has already abandoned it (the 2026-09-04 storm: 4h15m of +the accelerator at 100% on replies discarded as `unknown request id`). +Commands and model loads NEVER carry a deadline at any version — a shed +command desynchronizes model state. A host that declares nothing is an OLD +host and gets byte-identical v1 behaviour; skew in either direction is the +NORMAL state during a rolling deploy. + +Commands: load, unload, replace, reconfigure, status, uncache_frame, mem_stats. +`load`/`replace` compile on their own thread under a hard bound and never block +the loop; a load past the bound answers `reason: "compile-timeout"` and poisons +the worker (D653, see `ModelLoader`). +""" +from __future__ import annotations + +import asyncio +import concurrent.futures +import contextlib +import fcntl +import gc +import io +import json +import os +import platform +import re +import shutil +import struct +import sys +import threading +import time +from collections import deque +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable, Optional + +import numpy as np +from PIL import Image + +from postprocessors import POSTPROCESSORS + +# --------------------------------------------------------------------------- +# Wire protocol constants +# --------------------------------------------------------------------------- + +MSG_COMMAND = 0x00 +MSG_INFER_JPEG = 0x01 +MSG_INFER_RAW = 0x02 +# MSG_INFER_BATCH — N raw items packed into a single IPC frame. +# Wire format: +# [1B model_idx][1B count][N × item] +# Each item: +# [4B width][4B height][1B fmt][4B size][size bytes raw] +# Response payload: +# {"results": [, ...]} (same length as count) +# Each item is dispatched concurrently via asyncio.gather so the +# predict pool's existing parallelism applies; the saving over N +# separate calls is one IPC round-trip per item collapsed to one, +# with batch=4 semantics for fair benchmarking. +MSG_INFER_BATCH = 0x03 +MSG_CACHE_FRAME = 0x04 +MSG_INFER_CACHED = 0x05 + +RAW_FMT_RGB = 0x00 +RAW_FMT_BGR = 0x01 +RAW_FMT_GRAY = 0x02 + +# --------------------------------------------------------------------------- +# Wire-protocol versioning + the deadline on the wire (D350) +# --------------------------------------------------------------------------- + +# Highest wire version THIS worker implements. Reported in the ready reply; +# the effective version per connection is min() of both sides' declarations, +# so a worker and a host at different versions (normal during a rolling +# deploy) always agree on the layout. +PROTOCOL_VERSION = 2 + +# Opcodes that may carry (and be shed by) a deadline. Mirrors +# SHEDDABLE_MSG_TYPES in shared-inference-pool.ts — commands and cacheFrame +# are absent on BOTH sides: a shed command desynchronizes model state, a shed +# cacheFrame strands the inferCached that follows it. +SHEDDABLE_MSG_TYPES = frozenset({MSG_INFER_JPEG, MSG_INFER_RAW, MSG_INFER_BATCH, MSG_INFER_CACHED}) + +# Bytes of the absolute-deadline prefix a v2 sheddable payload carries. +DEADLINE_PREFIX_LEN = 8 + +# Sampling stride for the deadline-expired shed stderr line. Under a storm +# this fires at frame rate; one line per shed buried every other signal the +# last time a shed path logged unsampled (45 741 rows/hour, 2026-08-01). The +# counter (PoolCounters.shed_deadline_expired) keeps the exact total. +DEADLINE_SHED_LOG_EVERY = 500 + + +def _negotiate_wire_version(config: dict) -> int: + """Effective wire version for this connection: min(ours, host declared). + + No declaration (an OLD host) or a garbage value negotiates v1 — the + byte-identical historical layout. Pure; unit-tested. + """ + try: + declared = int(config.get("protocolVersion", 1) or 1) + except (TypeError, ValueError): + declared = 1 + return max(1, min(PROTOCOL_VERSION, declared)) + + +def _split_deadline(msg_type: int, payload: bytes, wire_version: int) -> "tuple[int, bytes]": + """Split a request payload into (deadline_ms, rest) by the NEGOTIATED + version. v1, and every non-sheddable opcode at any version, passes the + payload through untouched with deadline 0 (= no deadline, never sheds). + Raises ValueError on a v2 sheddable payload too short to carry its + prefix. Pure; unit-tested. + """ + if wire_version < 2 or msg_type not in SHEDDABLE_MSG_TYPES: + return 0, payload + if len(payload) < DEADLINE_PREFIX_LEN: + raise ValueError( + f"truncated deadline prefix on msg_type={msg_type} " + f"({len(payload)} bytes, need {DEADLINE_PREFIX_LEN})" + ) + (deadline_ms,) = struct.unpack(" bool: + """0 means "no deadline" and never expires (v1 host / unstamped).""" + return deadline_ms > 0 and now_ms > deadline_ms + + +def _dropped_deadline_payload() -> dict: + """The shed reply for an expired request — same shape the TS side already + recognises as flow control (`poolResultToEngineOutput`).""" + return { + "kind": "detections", + "detections": [], + "inferenceMs": 0, + "dropped": True, + "shedReason": "deadline-expired", + } + + +def _log_deadline_shed(total: int, model_idx: int, late_ms: float) -> None: + """Sampled stderr line for a deadline shed — first occurrence and every + DEADLINE_SHED_LOG_EVERY-th, carrying the running total so nothing is + lost, only repeated. A branch that drops work silently is how the + 2026-09-04 collapse produced not one line in four hours.""" + if total != 1 and total % DEADLINE_SHED_LOG_EVERY != 0: + return + sys.stderr.write( + f"deadline-expired shed: model {model_idx}, {late_ms:.0f}ms past deadline " + f"(total {total}, sampled every {DEADLINE_SHED_LOG_EVERY})\n" + ) + sys.stderr.flush() + + +# --------------------------------------------------------------------------- +# Preprocessing (unchanged from v1 — same shapes/math, moved into helpers) +# --------------------------------------------------------------------------- + + +def letterbox_image(img: Image.Image, size: int) -> tuple[Image.Image, float, tuple[int, int]]: + """Resize with letterbox padding; returns (PIL canvas, scale, (padX, padY)). + + Stays in uint8 PIL space — no float conversion. Used by the imageType + fast path where CoreML's predict() accepts a PIL Image directly and + handles the BGR/normalize/CVPixelBuffer dance internally. + """ + w, h = img.size + scale = size / max(w, h) + new_w = int(w * scale) + new_h = int(h * scale) + pad_x = (size - new_w) // 2 + pad_y = (size - new_h) // 2 + if w == size and h == size: + # Already at target size — skip resize + paste + alloc entirely. + return img, 1.0, (0, 0) + # A frameRef view already arrives at the model's content dimensions + # (e.g. 320x180 for a 16:9 YOLO-320 input). Add only the square padding; + # repeating the same-size geometric resize wastes CPU and can blur pixels. + resized = img if (w, h) == (new_w, new_h) else img.resize((new_w, new_h), Image.BILINEAR) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + canvas.paste(resized, (pad_x, pad_y)) + return canvas, scale, (pad_x, pad_y) + + +def letterbox(img: Image.Image, size: int) -> tuple[np.ndarray, float, tuple[int, int]]: + """Resize with letterbox padding; returns (array[0..1] HWC float32, scale, (padX, padY)). + + Used by multiArrayType inputs where the model expects a normalized + float tensor. Does the uint8→float32/255 conversion exactly once. + """ + canvas, scale, pad = letterbox_image(img, size) + arr = np.array(canvas, dtype=np.float32) / 255.0 + return arr, scale, pad + + +def resize_image(img: Image.Image, width: int, height: int) -> np.ndarray: + resized = img.resize((width, height), Image.BILINEAR) + return np.array(resized, dtype=np.float32) / 255.0 + + +def decode_jpeg(jpeg: bytes) -> Image.Image: + return Image.open(io.BytesIO(jpeg)).convert("RGB") + + +def wrap_raw(raw: bytes, width: int, height: int, fmt: int) -> Image.Image: + """Zero-copy (via PIL frombuffer) wrap of a raw frame buffer.""" + if fmt == RAW_FMT_GRAY: + img = Image.frombuffer("L", (width, height), raw, "raw", "L", 0, 1) + return img.convert("RGB") + mode = "BGR" if fmt == RAW_FMT_BGR else "RGB" + img = Image.frombuffer("RGB", (width, height), raw, "raw", mode, 0, 1) + return img + + +# --------------------------------------------------------------------------- +# Model slot + runtime-specific loading +# --------------------------------------------------------------------------- + + +@dataclass +class ModelSlot: + model: Any = None + config: dict = field(default_factory=dict) + loaded: bool = False + predict_fn: Optional[Callable[[dict], dict]] = None + # Atomic snapshot of model.spec input names; used by both single + # and batched predict paths to filter input_dicts against the + # CURRENTLY-loaded model. Defends against the race where an in- + # flight preprocess holds a reference to the previously-loaded + # model's `_input_names` and adds keys (iouThreshold, + # confidenceThreshold) that the new model rejects. + input_names: frozenset = field(default_factory=frozenset) + + +_runtime: str = "" +_runtime_lib: Any = None +# Max OPTIMAL_NUMBER_OF_INFER_REQUESTS across loaded OpenVINO models — used +# to size the predict pool so the per-model InferRequest pool is actually fed. +_ov_optimal_reqs: int = 0 +# Default OpenVINO predict-pool size when models load lazily (so we can't yet +# query the device's optimal request count). ≈ measured optimal on Intel +# CPU/iGPU/NPU (4-5). Threads idle-block on infer, so over-provisioning is cheap. +OV_DEFAULT_CONCURRENCY: int = 4 +# Floor for the InferRequest / predict-pool size when the device DOES answer. +# Under PERFORMANCE_HINT=LATENCY an Intel iGPU typically reports 1-2: one +# execution stream, and the extra requests buy pipelining the LATENCY hint is +# not asking for. Two rather than one so a second frame can be preprocessed +# while the first infers, without paying for four. +OV_MIN_CONCURRENCY: int = 2 + + +def _ov_request_count(optimal: int) -> int: + """How many InferRequests to create up front. + + ONE function because two call sites must agree: the per-model request pool + in `_load_model` and the predict-pool worker count in `_run`. They used to + compute it separately with the same expression, and the expression was + `max(optimal, OV_DEFAULT_CONCURRENCY)` — a FLOOR of four, not the fallback + its own comment described. On this hub that meant every model held four + GPU-resident InferRequests forever, per pool, while the device was asking + for one or two and `intel_gpu_top` showed the GPU 92% idle. Creating a + VAAPI device for an HEVC decode then failed with ENOMEM. + + `optimal <= 0` means the device could not be asked — models load lazily, + the property is missing — and UNKNOWN is not SMALL: that case keeps the + old default rather than guessing low. + """ + if optimal <= 0: + return OV_DEFAULT_CONCURRENCY + return max(optimal, OV_MIN_CONCURRENCY) +# OpenVINO device decision. AUTO is NEVER used: the AUTO plugin runs inference +# on a CPU "helper" while the GPU JIT-compiles, then hands the model over to the +# GPU mid-flight. InferRequests created during that window bind to the +# CPU-helper's Result ports; after handover the served model's ports diverge and +# req.infer() raises "Cannot find tensor for port opset1::Result score_8" (proven +# live on the N100 iGPU). Compiling ONE explicit device removes the handover +# entirely. We resolve an ordered candidate list (NPU > GPU > CPU) and compile the +# first that succeeds, with CPU as the guaranteed floor. +# +# OpenVINO PERFORMANCE_HINT is LATENCY: one deterministic execution stream. The +# THROUGHPUT hint multiplied memory and lengthened tail latency (an N100 +# OOM-freeze factor, see the ov_config note in _load_model); its env switch and +# the never-enabled AsyncInferQueue path were removed on 2026-10-06. +_OV_PERF_HINT: str = "LATENCY" + +# OpenVINO COMPILATION thread pool (CAMSTACK_OV_COMPILATION_THREADS, default 4). +# +# OpenVINO sizes the pool that COMPILES a model to the machine's core count, per +# compile, and TBB never retires those workers — they live as long as the +# process. A container cgroup counts THREADS, not processes, so every compile +# permanently spends `nproc` of the cgroup's pid budget. +# +# Measured on the hub (20 cores) 2026-09-18, READ-ONLY, before this bound: +# pids.current 991 / pids.max 2048 (22 fork refusals in 45 minutes earlier). +# The detection pool (pid 1183) held 214 threads 17 minutes after boot — the +# single largest thread holder in the container, 3x the next one (ffmpeg, 80). +# Thread start times cluster EXACTLY on its compile timestamps: 167 threads in +# a 4s boot burst (~8 models), then 47 more in a 2-model burst ten minutes +# later. 23 threads per compile on a 20-core box. ~10 models are resident. +# Expected after: ~4 threads per compile, i.e. ~40 instead of ~214 in that pool. +# +# Four, not one: the cold-compile storm (~25s for the full model set on an N100 +# iGPU) is where frames pile up, so the bound keeps some parallelism. It costs +# nothing at inference time — COMPILATION_NUM_THREADS is not +# INFERENCE_NUM_THREADS, which is deliberately left at OpenVINO's default (see +# `_ov_compilation_properties`). 0 disables the bound entirely. +try: + OV_COMPILATION_NUM_THREADS: int = int( + os.environ.get("CAMSTACK_OV_COMPILATION_THREADS", "4") + ) +except ValueError: + OV_COMPILATION_NUM_THREADS = 4 + + +def _ov_supported_properties(core: Any, device: str) -> list[str]: + """SUPPORTED_PROPERTIES for `device`, or [] when the device cannot be asked. + + A probe that FAILS knows nothing, and unknown is not "supported": the caller + then sends no extra property and compiles exactly as it always did. + """ + try: + return [str(p) for p in core.get_property(device, "SUPPORTED_PROPERTIES")] + except Exception: + return [] + + +def _ov_compilation_properties(supported: Any, threads: int) -> dict: + """The compile-time properties that BOUND OpenVINO's compilation pool. + + Only `COMPILATION_NUM_THREADS`, and only for a device that lists it. A + plugin REJECTS a property it does not support and the rejection surfaces as + a compile exception — which the failover loop in `_load_model` would read as + "this device cannot run the model" and quietly land a GPU model on the CPU. + Absent beats rejected. + + INFERENCE_NUM_THREADS is deliberately NOT set here. Under PERFORMANCE_HINT + =LATENCY the compiled model runs one stream, and the CPU device is this + pool's guaranteed fallback floor — capping its inference threads would trade + a pid saving we have not measured for a latency regression on exactly the + nodes that have no accelerator. The compilation pool is pure overhead; the + inference pool is the work. + """ + if threads <= 0 or "COMPILATION_NUM_THREADS" not in set(supported): + return {} + return {"COMPILATION_NUM_THREADS": threads} + + +# Inference precisions a catalog entry may require on an OpenVINO GPU. +_OV_GPU_PRECISIONS = ("f32", "f16") + + +def _ov_precision_properties(config: dict, device: str, supported: Any) -> dict: + """The INFERENCE_PRECISION_HINT a model's catalog entry requires on a GPU. + + An Intel GPU runs an FP32 IR at f16 by default. Some IRs do not survive + that: on 2026-10-07 the YOLO26 base IRs `yolo26s`, `yolo26m` and + `yolo26m-320` returned no valid detection on the Core Ultra iGPU and the + N100 (0 true / 146 false positives, or NaN scores) and matched the CPU at + f32. The entry says so (`gpuInferencePrecision`, threaded as config key + `gpuInferencePrecision`) — never a model-name match. + + GPU only: the NPU and CPU already compute these IRs correctly. And only for + a device that lists the key — a rejected property would make the failover + loop push the model onto the next device. + """ + precision = config.get("gpuInferencePrecision") + if precision not in _OV_GPU_PRECISIONS: + return {} + if not str(device).upper().startswith("GPU"): + return {} + if "INFERENCE_PRECISION_HINT" not in set(supported): + return {} + return {"INFERENCE_PRECISION_HINT": precision} + + +def _resolve_ov_devices(available: list[str], full_names: dict[str, str]) -> list[str]: + """Ordered EXPLICIT device candidates for compile-time failover (NPU>GPU>CPU). + + Never returns an AUTO/AUTO:.. mode (see the module comment above). `available` + is `core.available_devices`; `full_names` maps each device to its + FULL_DEVICE_NAME (missing entries = the probe failed, left unclassified). A + GPU is only accepted when its FULL_DEVICE_NAME probed successfully (this is + also how an Intel iGPU vs an NVIDIA dGPU is told apart downstream). Ordering: + NPU(s) -> GPU(s) -> CPU (always last). The compile loop tries each in order + and stops at the first that compiles; CPU is the guaranteed fallback. + """ + npus: list[str] = [] + gpus: list[str] = [] + for device in available: + full_device_name = full_names.get(device) + if "NPU" in device: + npus.append(device) + elif "GPU" in device and full_device_name is not None: + gpus.append(device) + ordered: list[str] = [*npus, *gpus] + if "CPU" in available and "CPU" not in ordered: + ordered.append("CPU") + if not ordered: + ordered = ["CPU"] + return ordered + + +# ONNX Runtime execution-provider names used by _resolve_onnx_providers. +ONNX_CUDA_EP = "CUDAExecutionProvider" +ONNX_COREML_EP = "CoreMLExecutionProvider" +ONNX_CPU_EP = "CPUExecutionProvider" + + +def _resolve_onnx_providers( + device: str, + plat: str, + machine: str, + available: list[str], + cuda_device_id: int = 0, +) -> list: + """Ordered ONNX Runtime providers — hardware first, CPU always last. + + Ordering for the AUTO/default case: + + darwin -> CoreMLExecutionProvider + linux/win on x86_64/AMD64 -> ("CUDAExecutionProvider", {"device_id": N}) + always -> CPUExecutionProvider appended LAST + + Deliberate adaptations for camstack: + - an operator device pin is honored: "cuda"/"coreml" put that EP first + (CPU still last as fallback); "cpu" compiles CPU only. + - hardware entries not present in `available` + (ort.get_available_providers()) are dropped — camstack ships the + plain `onnxruntime` wheel on most nodes, and requesting an EP the + build lacks raises at session creation instead of falling back. + - platform match uses startswith so a substring test on "win" never + wrongly matches "darwin". + """ + dev = (device or "").strip().lower() + ordered: list = [] + if dev == "cuda": + ordered.append((ONNX_CUDA_EP, {"device_id": cuda_device_id})) + elif dev == "coreml": + ordered.append(ONNX_COREML_EP) + elif dev == "cpu": + pass # explicit CPU pin — no hardware EPs + else: + # AUTO/default — platform-driven ordering. + if plat == "darwin": + ordered.append(ONNX_COREML_EP) + if plat.startswith(("linux", "win")) and machine in ("x86_64", "AMD64"): + ordered.append((ONNX_CUDA_EP, {"device_id": cuda_device_id})) + providers: list = [ + p for p in ordered + if (p[0] if isinstance(p, tuple) else p) in available + ] + providers.append(ONNX_CPU_EP) + return providers + + +# --------------------------------------------------------------------------- +# CoreML persistent compiled-model cache +# --------------------------------------------------------------------------- +# Known macOS behavior: the OS-level compiled-model cache is NOT reused across +# process restarts — every +# `ct.models.MLModel(.mlpackage)` load RECOMPILES the model, and the stale OS +# cache is only cleared on reboot. camstack loads the .mlpackage on every pool +# spawn, so each spawn paid the full recompile. Fix: after the first compile, +# copy the compiled `.mlmodelc` (get_compiled_model_path() returns a TEMP dir +# that dies with the MLModel object) into a persistent `.coreml-cache` dir next +# to the model (mirrors the OpenVINO `.ov-cache` convention); subsequent spawns +# load it via `ct.models.CompiledMLModel(...)` — no recompile. A stamp file +# fingerprinting the source model invalidates the cache when the model changes. +# HARD RULE: every failure in this machinery falls back to the plain +# `ct.models.MLModel(path)` load — exactly the pre-cache behavior. + +# --------------------------------------------------------------------------- +# Compile-cache lock — one worker compiles a model, the rest wait and HIT +# --------------------------------------------------------------------------- +# +# A pool is several python PROCESSES. They start together, all miss the +# compiled-model cache because nobody has written it yet, all pay the compile, +# and then all try to replace the same directory. Measured on the operator's +# Mac on 2026-09-19 over three hours: **59 cache HITs, 55 MISSes, 5 persist +# failures** — half the loads recompiling a model that was already on disk. +# +# The failures name a path INSIDE the bundle: +# +# CoreML: could not persist compiled-model cache +# ([Errno 2] No such file or directory: '.../yolov9m-320.mlmodelc/weights') +# +# which is `_persist_coreml_compiled` removing the old cache while another +# worker swaps its own copy into the same place: `rmtree` walks a directory and +# a child ceases to exist under it. +# +# So this lock is not belt-and-braces. It is what turns N compiles of one model +# into one, and what makes "replace the cache" single-writer. +# +# It is deliberately NOT fatal in any direction: an unusable lock directory, a +# platform without `flock`, or a holder that died mid-compile all end the same +# way — the caller compiles without the cache, which is exactly what it did +# before the cache existed. A cache is an optimisation, and an optimisation +# that can refuse to serve a camera is worse than every miss it prevents. + +# How long a waiter gives the compiling worker before compiling itself. +COMPILE_CACHE_LOCK_TIMEOUT_SEC = 180.0 +_COMPILE_CACHE_LOCK_POLL_SEC = 0.05 + + +@contextlib.contextmanager +def compile_cache_lock( + cache_root: str, + stem: str, + timeout_sec: float = COMPILE_CACHE_LOCK_TIMEOUT_SEC, +): + """Hold the per-MODEL compile lock, or yield False having failed to. + + Per model, not per cache: compiling the detector must not serialise behind + the embedder, which is a different file and a different compile. + """ + handle = None + held = False + try: + os.makedirs(cache_root, exist_ok=True) + handle = open(os.path.join(cache_root, f"{stem}.lock"), "a+") + deadline = time.monotonic() + max(0.0, timeout_sec) + while True: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + held = True + break + except OSError: + if time.monotonic() >= deadline: + break + time.sleep(_COMPILE_CACHE_LOCK_POLL_SEC) + except (OSError, AttributeError, NameError): + held = False + try: + yield held + finally: + if handle is not None: + try: + if held: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except OSError: + pass + try: + handle.close() + except OSError: + pass + + +COREML_CACHE_DIR_NAME = ".coreml-cache" +COREML_CACHE_STAMP_VERSION = 1 + + +def _coreml_cache_paths(model_path: str) -> tuple[str, str]: + """(compiled .mlmodelc cache dir, stamp file) for a source model path. + + The cache lives in a dot-dir next to the model file — same convention as + the OpenVINO `.ov-cache` — so it ships/wipes with the models dir. + """ + models_dir = os.path.dirname(os.path.abspath(model_path)) + stem = os.path.splitext(os.path.basename(model_path))[0] + cache_root = os.path.join(models_dir, COREML_CACHE_DIR_NAME) + return ( + os.path.join(cache_root, stem + ".mlmodelc"), + os.path.join(cache_root, stem + ".stamp.json"), + ) + + +def _coreml_model_fingerprint(model_path: str) -> dict: + """Cheap source-model fingerprint — file count + total size + max mtime. + + Works for both a plain file (.mlmodel) and a .mlpackage DIRECTORY (the + normal case), whose top-level mtime alone would not reflect content edits. + JSON-serializable so it round-trips through the stamp file unchanged. + + The package's top-level `Manifest.json` counts by size only, never by + mtime: coremltools rewrites it (same bytes) on EVERY open of the package — + `MLModel` and `utils.load_spec` alike (measured, coremltools 8.3, macOS + 26.2). The cache HIT path calls `load_spec`, so keyed on that mtime every + HIT invalidated the stamp for the next spawn: HIT, MISS, HIT, MISS. A + re-download rewrites the model and weights files too, so it still shows. + """ + if not os.path.isdir(model_path): + st = os.stat(model_path) + return {"kind": "file", "files": 1, "size": st.st_size, "mtimeNs": st.st_mtime_ns} + total_size = 0 + latest_mtime = 0 + file_count = 0 + manifest = os.path.join(model_path, "Manifest.json") + for root, _dirs, files in os.walk(model_path): + for name in files: + file_path = os.path.join(root, name) + st = os.stat(file_path) + total_size += st.st_size + file_count += 1 + if file_path == manifest: + continue + if st.st_mtime_ns > latest_mtime: + latest_mtime = st.st_mtime_ns + return {"kind": "dir", "files": file_count, "size": total_size, "mtimeNs": latest_mtime} + + +def _coreml_stamp_matches(stamp: Any, fingerprint: dict) -> bool: + """Pure decision: does a parsed stamp validate the cached compile?""" + return ( + isinstance(stamp, dict) + and stamp.get("version") == COREML_CACHE_STAMP_VERSION + and stamp.get("fingerprint") == fingerprint + ) + + +def _persist_coreml_compiled( + compiled_src: str, cache_path: str, stamp_path: str, fingerprint: dict, +) -> None: + """Copy coremltools' TEMP compiled dir into the persistent cache path. + + Crash-safe ordering: copy into a tmp sibling on the SAME filesystem, then + `os.replace` into place, and write the stamp LAST — a stamp file only ever + exists for a fully-moved `.mlmodelc`, so a crash mid-copy can never + produce a half-written cache that validates. Raises on failure — the + caller logs and continues WITHOUT a cache (model already loaded fine). + """ + cache_root = os.path.dirname(cache_path) + os.makedirs(cache_root, exist_ok=True) + tmp_dir = f"{cache_path}.tmp-{os.getpid()}" + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + try: + shutil.copytree(compiled_src, tmp_dir) + # Drop any stale previous cache: its stamp first (so nothing validates + # the old dir), then the dir itself — POSIX rename cannot replace a + # non-empty directory. + # Both removals tolerate "already gone". `compile_cache_lock` makes a + # concurrent swap rare, not impossible — a worker killed between taking + # the lock and the replace leaves precisely this state — and a previous + # cache that has ALREADY disappeared is the outcome we wanted anyway. + # Not tolerating it is what produced five + # `[Errno 2] ... .mlmodelc/weights` failures in three hours. + try: + os.remove(stamp_path) + except FileNotFoundError: + pass + shutil.rmtree(cache_path, ignore_errors=True) + os.replace(tmp_dir, cache_path) + except BaseException: + shutil.rmtree(tmp_dir, ignore_errors=True) + raise + stamp_tmp = f"{stamp_path}.tmp-{os.getpid()}" + try: + with open(stamp_tmp, "w", encoding="utf-8") as fh: + json.dump({"version": COREML_CACHE_STAMP_VERSION, "fingerprint": fingerprint}, fh) + os.replace(stamp_tmp, stamp_path) + except BaseException: + try: + os.remove(stamp_tmp) + except OSError: + pass + raise + + +def _try_coreml_cache_hit( + ct: Any, path: str, compute_units: Any, +) -> tuple[Any, Any] | None: + """(model, spec) from the persistent cache, or None for "compile it". + + Separate from the compile because it is asked TWICE: once before queueing + for the compile lock, and again once the lock is held — the second ask is + what turns every waiter's miss into a hit. + + `CompiledMLModel` has no `get_spec()`, so the spec is read straight from + the source .mlpackage with `ct.utils.load_spec` (cheap — parses the + protobuf, does not compile). Any failure answers None, which is the + pre-cache behaviour exactly. + """ + try: + cache_path, stamp_path = _coreml_cache_paths(path) + fingerprint = _coreml_model_fingerprint(path) + stamp: Any = None + if os.path.isfile(stamp_path): + with open(stamp_path, "r", encoding="utf-8") as fh: + stamp = json.load(fh) + if _coreml_stamp_matches(stamp, fingerprint) and os.path.isdir(cache_path): + spec = ct.utils.load_spec(path) + model = ct.models.CompiledMLModel(cache_path, compute_units) + sys.stderr.write(f"CoreML: compiled-model cache HIT {cache_path}\n") + sys.stderr.flush() + return model, spec + except Exception as exc: + sys.stderr.write( + f"CoreML: compiled-model cache lookup failed ({exc}); " + f"falling back to plain compile of {os.path.basename(path)}\n" + ) + sys.stderr.flush() + return None + + +def _acquire_coreml_model(ct: Any, path: str, compute_units: Any) -> tuple[Any, Any]: + """Return (model, spec) for the CoreML branch via the persistent cache. + + Cache HIT → `CompiledMLModel(cache)`, no recompile (`_try_coreml_cache_hit`). + Cache MISS → take the per-model compile lock, ask AGAIN (the winner may + have finished while we queued), then `MLModel(path)` — the + pre-cache behaviour — and persist it for the next spawn. + LOCK LOST → compile anyway, serve the model, write nothing. + ANY failure in the cache machinery → the plain MLModel path, exactly as + before this cache existed. Never raises differently from pre-cache code. + """ + hit = _try_coreml_cache_hit(ct, path, compute_units) + if hit is not None: + return hit + + # A MISS here is not yet a compile. The pool starts its workers together, + # so every one of them arrives at this line at the same instant with the + # same answer — measured 55 misses against 59 hits on the Mac, which is + # not a cold cache, it is N workers each compiling what the others are + # compiling. One of them takes the lock; the rest wait and then find the + # HIT the winner just wrote. + cache_root = os.path.dirname(_coreml_cache_paths(path)[0]) + stem = os.path.splitext(os.path.basename(path))[0] + with compile_cache_lock(cache_root, stem) as held: + if held: + hit = _try_coreml_cache_hit(ct, path, compute_units) + if hit is not None: + return hit + + model = ct.models.MLModel(path, compute_units=compute_units) + spec = model.get_spec() + + # Only the lock HOLDER writes the cache. A worker that compiled + # because it could not get the lock has a perfectly good model and no + # business replacing a directory somebody else owns — that swap is the + # race, and skipping it costs nothing: the holder is writing the same + # bytes. + if not held: + sys.stderr.write( + f"CoreML: compiled without the cache lock — compiled " + f"{os.path.basename(path)} and left the cache to its holder\n" + ) + sys.stderr.flush() + return model, spec + + try: + cache_path, stamp_path = _coreml_cache_paths(path) + fingerprint = _coreml_model_fingerprint(path) + compiled_src = model.get_compiled_model_path() + _persist_coreml_compiled(compiled_src, cache_path, stamp_path, fingerprint) + sys.stderr.write( + f"CoreML: compiled-model cache MISS — compiled and persisted {cache_path}\n" + ) + sys.stderr.flush() + except Exception as exc: + sys.stderr.write( + f"CoreML: could not persist compiled-model cache ({exc}); " + f"continuing without cache\n" + ) + sys.stderr.flush() + return model, spec + + +def _init_runtime(runtime: str, device: str = "") -> None: + global _runtime, _runtime_lib + _runtime = runtime + if runtime == "coreml": + import coremltools as ct + _runtime_lib = ct + elif runtime == "openvino": + # `openvino.runtime` is deprecated (removed in a future OpenVINO release); + # `Core` is exported from the top-level `openvino` namespace. + from openvino import Core + _runtime_lib = Core() + elif runtime == "onnxruntime": + import onnxruntime as ort + _runtime_lib = ort + elif runtime == "edgetpu": + # Coral USB Edge TPU via LiteRT (ai-edge-litert) + the libedgetpu delegate. + # The delegate is created ONCE here and shared across all model slots (the + # USB device is a single hardware context). See docs: the delegate's TF + # version MUST match the interpreter — feranick libedgetpu tf2.17.1 pairs + # with ai-edge-litert 2.1.6. Install: /data/deps/edgetpu-install.sh. + import threading + from ai_edge_litert.interpreter import Interpreter, load_delegate + lib_path = os.environ.get( + "CAMSTACK_EDGETPU_LIB", "/data/deps/edgetpu/lib/libedgetpu.so.1", + ) + # Multiple Corals coexist and run concurrently, one pool each. libedgetpu + # addresses a specific unit by an ordinal device string ("usb", "usb:1", + # "pci:0", …); the orchestrator passes it as the pool `device`. Omitted / + # bare "usb" ⇒ the first available Coral (single-Coral default). + dev = (device or "").strip() + opts = {"device": dev} if dev and dev not in ("usb", "all", "default") else None + # The Coral is a SINGLE-CONTEXT USB device: only ONE process may hold the + # libedgetpu delegate at a time. When a prior pool (e.g. a previous runner + # generation, or a just-disposed device pool) is still releasing the USB, + # a fresh load_delegate fails transiently. Retry with backoff instead of + # hard-failing the whole pool — the device frees within a second or two of + # the old holder exiting. (A genuinely-held device still fails after the + # retries, with the real libedgetpu error surfaced.) + import time as _time + last_err = None + delegate = None + for attempt in range(6): + try: + delegate = load_delegate(lib_path, opts) if opts else load_delegate(lib_path) + break + except Exception as e: # noqa: BLE001 — surface the real error after retries + last_err = e + sys.stderr.write( + f"EdgeTPU: delegate load attempt {attempt + 1}/6 failed " + f"(device={dev or 'usb'}): {e}\n" + ) + sys.stderr.flush() + _time.sleep(0.75 * (attempt + 1)) + if delegate is None: + raise ValueError( + f"EdgeTPU: delegate not available after 6 attempts on device " + f"'{dev or 'usb'}' — the Coral is held by another process or " + f"absent. Last error: {last_err}" + ) + sys.stderr.write(f"EdgeTPU: delegate bound to device '{dev or 'usb'}'\n") + _runtime_lib = { + "Interpreter": Interpreter, + "delegate": delegate, + # invoke() on one interpreter is not re-entrant, and the EdgeTPU is a + # single-context device — serialise all predicts through one lock. + "lock": threading.Lock(), + } + sys.stderr.write(f"EdgeTPU: delegate loaded from {lib_path}\n") + sys.stderr.flush() + else: + raise ValueError(f"Unknown runtime: {runtime}") + + +# --------------------------------------------------------------------------- +# Batch capability is a HARDWARE FACT, not a setting (D202) +# --------------------------------------------------------------------------- +# Whether a compiled graph may keep a DYNAMIC batch axis is decided here, from +# the (engine, device) pair, at the moment of load — never per model, never by +# an operator, never in a conversion script. The conversion scripts and the +# model-studio convert path both emit dynamic-batch graphs (`ov.convert_model` +# preserves the ONNX dynamic axis), and the same artifact is loaded on CPU, iGPU, +# NPU and ANE nodes — so the artifact cannot know, and only the loader does. +# +# The one target that refuses: the Intel NPU. Its Level-Zero compiler rejects a +# dynamic dimension outright ("Got negative shape dim bound: '-1'", then +# ZE_RESULT_ERROR_INVALID_ARGUMENT). Found live 2026-08-20 on the hub, and the +# symptom is the trap — a pool load failure is a RETRY, not an error the caller +# sees, so an NPU-pinned node repeated "Loading step variant" forever and looked +# busy rather than broken. Under AUTO it is quieter and just as wrong: the NPU +# candidate silently loses every dynamic-batch model to the GPU fallback. +# +# This is COMPILE shape only. The pool's runtime request grouping (`batch_mode` +# / `max_batch_size`, and the CoreML stacked-tensor fast path gated on the +# export's own RangeDim) is a separate, untouched optimisation. +STATIC_BATCH_PIN = 1 + + +def _static_batch_required(runtime: str, device: str) -> bool: + """THE authority: does this (engine, device) pair refuse a dynamic shape? + + Pure — no OpenVINO import, so it is unit-testable in a bare sandbox. + + openvino + NPU -> True (Level-Zero rejects any dynamic dim) + openvino + CPU/GPU -> False + onnxruntime -> False (every EP we ship handles dynamic axes) + coreml -> False (a RangeDim export is exactly what the batched + ANE dispatch path reads; pinning would delete it) + edgetpu -> False (a tflite graph is static by construction) + """ + if runtime != "openvino": + return False + return device.strip().upper().startswith("NPU") + + +def _static_batch_plan( + shapes: "list[list[int | None] | None]", +) -> "tuple[dict[int, list[int]], list[int]]": + """Pure. Given each input's dims (``None`` = dynamic axis, ``None`` for the + whole entry = unreadable), return ``(plan, unpinnable)``: + + plan {input_index: static_dims} for inputs whose BATCH axis is + dynamic and whose every other axis is already static. + unpinnable [input_index] for inputs carrying a dynamic axis that is NOT + the batch — there is no value we are allowed to invent for a + spatial dim, so the caller compiles as-is and SAYS SO rather + than letting the failure be silent. + """ + plan: "dict[int, list[int]]" = {} + unpinnable: "list[int]" = [] + for index, dims in enumerate(shapes): + if dims is None: + unpinnable.append(index) + continue + if len(dims) == 0: + continue + if any(d is None for d in dims[1:]): + unpinnable.append(index) + continue + if dims[0] is not None: + continue # already a static batch — the rule pins dynamic axes only + plan[index] = [STATIC_BATCH_PIN] + [int(d) for d in dims[1:] if d is not None] + return plan, unpinnable + + +def _pin_batch_for_device(core: Any, source: Any, path: str, device: str) -> Any: + """OpenVINO applicator of {@link _static_batch_required}. + + Returns what `compile_model` should be handed for `device`: the untouched + `source` when the target accepts a dynamic shape (or the graph is already + static), otherwise a model RESHAPED to a static batch of + {@link STATIC_BATCH_PIN}. `source` is either the IR path (the plain load + path) or an already-built model object (the PrePostProcessor path); the pin + is applied to whichever, and it only ever touches axis 0, so a PPP model's + rewritten uint8 NHWC input survives it. Every outcome is logged — the + derivation must be readable in the pool stderr. + + (`_ov_dim_to_int` is defined further down with the other OV shape helpers.) + """ + name = os.path.basename(path) + if not _static_batch_required("openvino", device): + return source + try: + model = core.read_model(path) if isinstance(source, str) else source + except Exception as exc: + sys.stderr.write( + f"OpenVINO: {device} needs a static shape but re-reading {name} failed " + f"({exc}); compiling the source as-is\n" + ) + sys.stderr.flush() + return source + + shapes: "list[list[int | None] | None]" = [] + for inp in model.inputs: + try: + pshape = inp.get_partial_shape() + shapes.append([_ov_dim_to_int(pshape[i]) for i in range(len(pshape))]) + except Exception: + shapes.append(None) + + plan, unpinnable = _static_batch_plan(shapes) + if unpinnable: + sys.stderr.write( + f"OpenVINO: {name} has a DYNAMIC NON-BATCH axis on input(s) {unpinnable} " + f"and {device} refuses dynamic dims — no value may be invented for a " + f"spatial axis, so this compile is expected to fail\n" + ) + sys.stderr.flush() + if not plan: + sys.stderr.write( + f"OpenVINO: batch shape for {name} on {device} — already static, no " + f"reshape (derived from engine+device)\n" + ) + sys.stderr.flush() + return source + try: + from openvino import PartialShape + + # `Model.reshape` mutates IN PLACE, and on the PrePostProcessor path + # `source` IS the model object the compile loop reuses for the NEXT + # candidate. Pinning it directly would leak this NPU-only decision into + # the GPU/CPU fallback — a candidate that accepts a dynamic shape would + # silently get the pinned graph. So reshape a CLONE and leave `source` + # exactly as the caller built it. + model = model.clone() if not isinstance(source, str) else model + model.reshape({index: PartialShape(dims) for index, dims in plan.items()}) + except Exception as exc: + sys.stderr.write( + f"OpenVINO: static-batch reshape of {name} for {device} failed ({exc}); " + f"compiling the source as-is\n" + ) + sys.stderr.flush() + return source + sys.stderr.write( + f"OpenVINO: batch shape for {name} on {device} — PINNED to " + f"{STATIC_BATCH_PIN} on input(s) {sorted(plan)} because the target refuses " + f"a dynamic dim (derived from engine+device, not configured)\n" + ) + sys.stderr.flush() + return model + + +def _load_model(slot: ModelSlot, config: dict) -> None: + """Load a model into a slot using the active runtime. Thin adapter per backend.""" + slot.config = dict(config) + path = config["path"] + + if _runtime == "coreml": + ct = _runtime_lib + device = config.get("device", "all") + compute_map = { + "cpu": ct.ComputeUnit.CPU_ONLY, + "gpu": ct.ComputeUnit.CPU_AND_GPU, + "ane": ct.ComputeUnit.CPU_AND_NE, + "all": ct.ComputeUnit.ALL, + } + # NOTE: a macOS behavior can cause the .mlpackage to be RECOMPILED on + # every load with the compiled cache not reused until reboot. camstack + # loads the .mlpackage on + # every pool spawn, so it was exposed to the same trap. + # _acquire_coreml_model routes through a persistent `.coreml-cache` + # of the compiled `.mlmodelc` (CompiledMLModel on hit, plain MLModel + # compile + persist on miss) and returns the spec read via + # ct.utils.load_spec on hits — falling back to the plain + # MLModel/get_spec() path on ANY cache failure. + # + # First, the package is loaded once in a CHILD process: a package that + # aborts Core ML's native loader (SIGABRT, which Python cannot catch) + # is refused here with `CoremlLoadCrashed` instead of killing this + # worker and, through the respawn, every worker after it. + import coreml_load_probe + + coreml_load_probe.ensure_coreml_loadable(path, device) + model, spec = _acquire_coreml_model( + ct, path, compute_map.get(device, ct.ComputeUnit.ALL), + ) + # Report the compute unit that was ACTUALLY applied so a silent + # CPU-only placement is visible in the pool stderr (mirrors the + # OpenVINO EXECUTION_DEVICES / ONNX provider reports). Both MLModel + # and CompiledMLModel expose `compute_unit`. + _actual_cu = getattr(model, "compute_unit", None) + sys.stderr.write( + f"CoreML: loaded {os.path.basename(path)} device={device} " + f"compute_unit={_actual_cu}\n" + ) + sys.stderr.flush() + input_spec = spec.description.input[0] + slot.config["_input_name"] = input_spec.name + slot.config["_input_type"] = input_spec.type.WhichOneof("Type") + slot.config["_input_names"] = [i.name for i in spec.description.input] + if slot.config["_input_type"] == "multiArrayType": + slot.config["_input_shape"] = list(input_spec.type.multiArrayType.shape) + # Detect batch flexibility — when the model's first axis is + # declared as a `shapeRange` (RangeDim at export time), we + # can stack N preprocessed frames into a single (N,3,H,W) + # tensor and issue ONE predict call instead of iterating + # the list-of-dicts path. One ANE dispatch per batch. + mat = input_spec.type.multiArrayType + if mat.WhichOneof("ShapeFlexibility") == "shapeRange" and len(mat.shapeRange.sizeRanges) > 0: + batch_axis = mat.shapeRange.sizeRanges[0] + if batch_axis.upperBound > 1: + slot.config["_supports_batch"] = True + slot.config["_max_batch"] = int(batch_axis.upperBound) + slot.model = model + slot.input_names = frozenset(i.name for i in spec.description.input) + # Filter via slot.input_names atomically read from the CURRENT + # slot. During a model `replace`, in-flight preprocess calls + # may have added stale keys (iouThreshold, confidenceThreshold) + # for the prior model that the new one rejects. + def _predict(inp: dict, _slot: ModelSlot = slot) -> dict: + names = _slot.input_names + filtered = {k: v for k, v in inp.items() if k in names} if names else inp + return _slot.model.predict(filtered) + slot.predict_fn = _predict + + elif _runtime == "openvino": + core = _runtime_lib + ov_device_req = str(config.get("device", "AUTO")).upper() + # Device switch (see _resolve_ov_devices above): an AUTO/default request + # classifies the enumerated devices by FULL_DEVICE_NAME and yields an + # ORDERED EXPLICIT candidate list (NPU>GPU>CPU) for compile-time + # failover. An operator-pinned device (config "device" = gpu/cpu/npu) is + # honored unchanged — compiled directly, no failover. + if ov_device_req in ("AUTO", "DEFAULT", ""): + available = list(core.available_devices) + full_names: dict[str, str] = {} + for dev in available: + try: + full_names[dev] = str(core.get_property(dev, "FULL_DEVICE_NAME")) + except Exception: + pass + candidates = _resolve_ov_devices(available, full_names) + sys.stderr.write( + f"OpenVINO: explicit device candidates {candidates} " + f"(available: {available}, names: {full_names})\n" + ) + sys.stderr.flush() + else: + candidates = [ov_device_req] # operator pin — compiled directly + sys.stderr.write(f"OpenVINO: operator-pinned device {ov_device_req}\n") + sys.stderr.flush() + # PERFORMANCE_HINT is LATENCY: one + # deterministic execution stream instead of N. THROUGHPUT spins up + # multiple internal streams whose lazy request creation historically + # collided with the AUTO handover AND multiplied memory (a factor in the + # N100 OOM-freeze); it also RAISES OPTIMAL_NUMBER_OF_INFER_REQUESTS, which + # enlarges the request pool at a memory / tail-latency cost. + # LATENCY keeps a single stream; camera-level concurrency comes from the + # fixed InferRequest pool built below. + ov_config = {"PERFORMANCE_HINT": _OV_PERF_HINT} + # Persist compiled GPU/NPU kernels so the multi-second JIT compile is paid + # once (first boot) instead of on every pool spawn. The cold compile storm + # (~25s for the full model set on an N100 iGPU) is where frames pile up; + # caching the kernels shrinks that window on subsequent boots. + try: + _cache_dir = os.path.join(os.path.dirname(os.path.abspath(path)), ".ov-cache") + os.makedirs(_cache_dir, exist_ok=True) + ov_config["CACHE_DIR"] = _cache_dir + except OSError: + pass + # Fix 2 — fold uint8→float/scale/layout preprocessing INTO the compiled + # graph via PrePostProcessor. When it succeeds we compile the PPP model + # object (fed uint8 NHWC at inference time); on ANY failure we fall back + # to compiling the plain `path` (byte-identical to the historical float + # preprocess). Always on for OpenVINO (tests flip `_OV_PPP_ENABLED`). + ppp_meta: Optional[dict] = None + compile_source: Any = path + if _OV_PPP_ENABLED: + try: + ppp_model, ppp_meta = _build_ov_ppp_model(core, path, config) + if ppp_model is not None: + compile_source = ppp_model + sys.stderr.write( + f"OpenVINO: PrePostProcessor folded for {os.path.basename(path)} " + f"(uint8 NHWC in, {ppp_meta}); Python preprocess stays uint8\n" + ) + sys.stderr.flush() + except Exception as exc: + ppp_meta = None + compile_source = path + sys.stderr.write( + f"OpenVINO: PrePostProcessor setup failed for {os.path.basename(path)} " + f"({exc}); falling back to Python float preprocess\n" + ) + sys.stderr.flush() + # Compile-time failover over the explicit candidate list (NO AUTO): try + # each device in order (NPU>GPU>CPU) and stop at the first that compiles. + # CPU is the guaranteed floor. If every candidate fails, re-raise — the + # existing load/replace error handler reports the failure. + compiled = None + ov_device = None + # What the compilation bound actually became on the device that won — + # 0 means the device does not expose the key, so this compile spent + # `nproc` threads for the life of the process and the operator must be + # able to SEE that rather than assume the bound applied. + ov_compile_threads = 0 + ov_precision = "device-default" + last_exc: Optional[BaseException] = None + for candidate in candidates: + try: + # Batch capability is derived from THIS candidate, not from the + # artifact and not from a setting (D202): a graph with a dynamic + # batch axis is reshaped static before it can reach a compiler + # that refuses dynamic dims (the Intel NPU), and is handed over + # untouched to every other target. + candidate_source = _pin_batch_for_device( + core, compile_source, path, candidate, + ) + # Bound the per-compile TBB pool this candidate would otherwise + # size to `nproc` and never give back (see + # OV_COMPILATION_NUM_THREADS). Gated on the device's own + # SUPPORTED_PROPERTIES so a plugin that does not know the key + # compiles unchanged instead of failing over to the CPU. + candidate_supported = _ov_supported_properties(core, candidate) + candidate_config = { + **ov_config, + **_ov_compilation_properties( + candidate_supported, + OV_COMPILATION_NUM_THREADS, + ), + # A model the GPU must not run at its default f16 + # (catalog `gpuInferencePrecision`). + **_ov_precision_properties(config, candidate, candidate_supported), + } + compiled = core.compile_model( + candidate_source, device_name=candidate, config=candidate_config, + ) + ov_device = candidate + ov_compile_threads = int( + candidate_config.get("COMPILATION_NUM_THREADS", 0) + ) + ov_precision = str( + candidate_config.get("INFERENCE_PRECISION_HINT", "device-default") + ) + break + except Exception as exc: + last_exc = exc + sys.stderr.write(f"OpenVINO: compile on {candidate} failed: {exc}\n") + if candidate != candidates[-1]: + sys.stderr.write("OpenVINO: falling back to next device.\n") + sys.stderr.flush() + if compiled is None: + if last_exc is not None: + raise last_exc + raise RuntimeError("OpenVINO: no device available to compile the model") + # Name the resolved mode + what OpenVINO actually placed the model on + # (the EXECUTION_DEVICES property) so a silent CPU + # placement is visible in the pool stderr. + try: + _exec_devices = compiled.get_property("EXECUTION_DEVICES") + except Exception: + _exec_devices = "unknown" + sys.stderr.write( + f"OpenVINO: compiled {os.path.basename(path)} mode={ov_device} " + f"EXECUTION_DEVICES={_exec_devices} compilationThreads=" + f"{ov_compile_threads if ov_compile_threads > 0 else 'unbounded(device rejects the key)'}" + f" inferencePrecision={ov_precision}\n" + ) + # A model that needs a GPU precision the device would not take runs at + # f16 and may return garbage — say so rather than look healthy. + if ( + config.get("gpuInferencePrecision") in _OV_GPU_PRECISIONS + and str(ov_device).upper().startswith("GPU") + and ov_precision == "device-default" + ): + sys.stderr.write( + f"OpenVINO: WARNING {os.path.basename(path)} requires " + f"inferencePrecision={config.get('gpuInferencePrecision')} on {ov_device}, " + "which does not list INFERENCE_PRECISION_HINT — it runs at the device default " + "and its detections may be invalid\n" + ) + sys.stderr.flush() + # An OpenVINO IR converted from ONNX can carry an UNNAMED output tensor + # (e.g. `yolov8n-package`): `get_any_name()` then raises + # "Attempt to get a name for a Tensor without names", which previously + # failed the whole model load on OpenVINO nodes (the package-detection + # step could never run on OpenVINO). Outputs are fetched POSITIONALLY in + # `predict` below, so a synthetic key per unnamed port is sufficient. + output_names = [] + for _i, _o in enumerate(compiled.outputs): + try: + output_names.append(_o.get_any_name()) + except Exception: + output_names.append(f"output{_i}") + + # Record the device's optimal infer-request count so the dispatcher + # can size the predict pool to actually feed the streams. + global _ov_optimal_reqs + try: + opt = int(compiled.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS")) + if opt > _ov_optimal_reqs: + _ov_optimal_reqs = opt + except Exception: + pass + + # Size N ONCE from the device's optimal infer-request count (falling back + # to OV_DEFAULT_CONCURRENCY only when the property is missing/zero) so it + # matches the predict-pool worker count sized in _run(); `_ov_request_count` + # is the one place it is computed. + import queue as _queue + + try: + _n_req = int(compiled.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS")) + except Exception: + _n_req = 0 + _n_req = _ov_request_count(_n_req) + + # Create ALL InferRequests ONCE, now, post-compile. With AUTO gone there is no CPU->GPU handover, so no request + # can be born mid-handover and bind to stale ports — the score_8 race + # is structurally removed. Requests are handed out through a thread- + # safe queue: an InferRequest is never touched by two threads at once + # (checkout/return), so sharing a fixed pool is safe and avoids the old + # thread-local lazy creation. + _req_pool: "_queue.Queue" = _queue.Queue() + for _ in range(_n_req): + _req_pool.put(compiled.create_infer_request()) + sys.stderr.write(f"OpenVINO: created {_n_req} InferRequest(s) up-front on {ov_device}\n") + sys.stderr.flush() + + def predict(inp_dict: dict, _names=output_names, _pool=_req_pool) -> dict: + inp = list(inp_dict.values())[0] + req = _pool.get() + try: + result = req.infer(inp) + finally: + _pool.put(req) + # Output fetch by positional index — stable and harmless now that + # the ports no longer diverge (single explicit device, no handover). + return {name: result[i] for i, name in enumerate(_names)} + + slot.model = compiled + slot.predict_fn = predict + # When the PPP preprocessor is folded in, `_preprocess` produces uint8 + # NHWC and short-circuits BEFORE the NHWC/NCHW float logic — so the + # `_input_shape` heuristic below is irrelevant for PPP models. Record + # the plan so `_preprocess` takes the uint8 fast path. + if ppp_meta is not None: + slot.config["_ov_ppp"] = ppp_meta + else: + # Capture the model's declared input layout so `_preprocess` feeds + # NHWC vs NCHW correctly (ArcFace/CLIP are NHWC). Best-effort — a + # failure leaves `_input_shape` unset → NCHW fallback (unchanged). + try: + slot.config["_input_shape"] = list(compiled.inputs[0].get_partial_shape()) + except Exception: + pass + + elif _runtime == "onnxruntime": + ort = _runtime_lib + ort_device = str(config.get("device", "") or "") + # Ordered providers — hardware EP first, CPU LAST as + # fallback (see _resolve_onnx_providers): a "cuda"/"coreml" pin puts + # that EP first, a "cpu" pin compiles CPU only, and the AUTO/default + # case derives the hardware EP from the platform. + try: + available_eps = list(ort.get_available_providers()) + except Exception: + available_eps = [ONNX_CPU_EP] + providers = _resolve_onnx_providers( + ort_device, sys.platform, platform.machine(), available_eps, + ) + session = ort.InferenceSession(path, providers=providers) + # Report the EP that ACTUALLY initialized (CPU stripped from the log) + # so a silent CPU fallback is visible. + _active_eps = [p for p in session.get_providers() if p != ONNX_CPU_EP] + _requested = [p[0] if isinstance(p, tuple) else p for p in providers] + sys.stderr.write( + f"ONNX: execution provider {', '.join(_active_eps) or ONNX_CPU_EP} " + f"(device={ort_device or 'auto'}, requested={_requested}, " + f"available={available_eps})\n" + ) + sys.stderr.flush() + output_names = [o.name for o in session.get_outputs()] + slot.config["_input_name"] = session.get_inputs()[0].name + # Declared input layout (ArcFace/CLIP are NHWC) → correct preprocess. + try: + slot.config["_input_shape"] = session.get_inputs()[0].shape + except Exception: + pass + # Declared input DTYPE. super-gradients / Frigate YOLO-NAS exports bake + # their preprocessing into the graph and take `tensor(uint8)` — the + # preprocess must feed raw uint8 pixels, not the float32 [0,1] tensor + # (a hard onnxruntime type error otherwise). + try: + if session.get_inputs()[0].type == "tensor(uint8)": + slot.config["_input_dtype"] = "uint8" + except Exception: + pass + + def predict(inp_dict: dict) -> dict: + outputs = session.run(output_names, inp_dict) + return {name: out for name, out in zip(output_names, outputs)} + + slot.model = session + slot.predict_fn = predict + + elif _runtime == "edgetpu": + Interpreter = _runtime_lib["Interpreter"] + delegate = _runtime_lib["delegate"] + lock = _runtime_lib["lock"] + interp = Interpreter(path, experimental_delegates=[delegate]) + interp.allocate_tensors() + in_det = interp.get_input_details()[0] + out_dets = interp.get_output_details() + _, in_h, in_w, _ = (int(x) for x in in_det["shape"]) + in_dtype = np.dtype(in_det["dtype"]).name # 'uint8' | 'int8' | 'float32' + slot.config["_input_name"] = in_det["name"] + slot.config["_edgetpu_in_dtype"] = in_dtype + slot.config["inputWidth"] = in_w + slot.config["inputHeight"] = in_h + # The FILE's input size is the truth for a compiled Edge TPU graph: a + # Coral build may be compiled at a smaller input than the catalog + # entry's other formats (activations share the 8 MB on-chip memory + # with the parameter cache, D710), and a decoder sized from the + # catalog would put every prior in the wrong cell. + slot.config["inputSize"] = max(in_w, in_h) + in_index = int(in_det["index"]) + out_index = [int(o["index"]) for o in out_dets] + from edgetpu_dequant import edgetpu_output_keys + + out_keys = edgetpu_output_keys( + [str(o.get("name") or "") for o in out_dets], slot.config.get("postprocessor") + ) + sys.stderr.write( + f"EdgeTPU: loaded {os.path.basename(path)} in={in_w}x{in_h} " + f"dtype={in_dtype} outputs={len(out_dets)}\n" + ) + sys.stderr.flush() + + def predict( + inp_dict: dict, + _interp=interp, _lock=lock, _in=in_index, _outs=out_index, _details=out_dets, + _keys=out_keys, + ) -> dict: + from edgetpu_dequant import dequant_edgetpu_output + + tensor = next(iter(inp_dict.values())) + with _lock: + _interp.set_tensor(_in, tensor) + _interp.invoke() + # `get_tensor` returns a copy, so the Coral is released before + # the dequantisation below runs. + raws = [_interp.get_tensor(idx) for idx in _outs] + # Positional keys — ssd reads "0".."3"; yolo-split-dfl pairs by + # shape — except for a postprocessor that matches by name + # (`edgetpu_output_keys`). Integer outputs are dequantised here so + # every postprocessor sees float32 (SSD Detection_PostProcess is + # already float and passes through). + result = {} + for i, raw in enumerate(raws): + detail = _details[i] + q = detail.get("quantization") or (0.0, 0) + scale = float(q[0]) if q else 0.0 + zp = int(q[1]) if q else 0 + dtype_name = str(np.dtype(detail["dtype"]).name) + result[_keys[i]] = dequant_edgetpu_output( + raw, dtype_name=dtype_name, scale=scale, zero_point=zp + ) + return result + + slot.model = interp + slot.predict_fn = predict + + slot.loaded = True + + +def _unload_model(slot: ModelSlot) -> None: + slot.model = None + slot.predict_fn = None + slot.input_names = frozenset() + slot.config = {} + slot.loaded = False + + +# --------------------------------------------------------------------------- +# Preprocess + run_inference — pure sync helpers called from worker threads +# --------------------------------------------------------------------------- + + +# Preprocess cache for bench frames only. Keyed by (_bench_frame_id, inputSize) +# where _bench_frame_id is a stable integer tag set on PIL Images stored in +# frame_cache. Camera frames don't have this tag → no cache → correct behavior. +# Avoids the CPython id() reuse problem that caused stale detections. +_bench_preprocess_cache: dict[tuple[int, int], tuple[dict, float, tuple[int, int]]] = {} + +# Verifiable proof that the bench preprocess cache is actually HIT on the +# sustained-throughput BATCH path. A tagged frame that misses runs the FULL +# decode-independent preprocess (letterbox + float/255 + transpose); a hit +# returns the pre-built input tensor untouched — pure inference, no CPU +# preprocess. The historical bug was that MSG_INFER_BATCH wrapped each item +# fresh WITHOUT the tag, so the cache was NEVER hit on the batch path and every +# inference re-ran the full preprocess. The sustained benchmark now asserts +# hits ≫ misses (surfaced in the `status` command) so that regression is caught. +_bench_cache_hits: int = 0 +_bench_cache_misses: int = 0 + + +def _bench_cache_stats() -> dict: + """(hits, misses) for the bench preprocess cache — surfaced in `status`.""" + return {"hits": _bench_cache_hits, "misses": _bench_cache_misses} + +def _is_channels_last(input_shape) -> bool: + """True when a 4-D model input is NHWC ([N,H,W,C]) rather than NCHW + ([N,C,H,W]). Heuristic: the LAST dim is a channel count (1 or 3) and the + SECOND dim is a concrete non-channel value. ArcFace / CLIP declare + [N,112,112,3] → True; standard detectors [N,3,H,W] → False. Robust to + dynamic/None dims → returns False (NCHW, the prior unconditional + behaviour) whenever it can't be sure, so detectors are never touched.""" + if input_shape is None or len(input_shape) != 4: + return False + + def _as_int(v) -> "int | None": + try: + return int(v) + except (TypeError, ValueError): + pass + # OpenVINO `Dimension` objects don't support int() (TypeError even + # for STATIC dims — verified live on the hub: arcface's captured + # shape [?,112,112,3] read as all-None → NCHW fallback → NPU shape + # error). Static Dimensions expose get_length(); dynamic stay None. + try: + if getattr(v, "is_static", False): + return int(v.get_length()) + except (TypeError, ValueError, AttributeError): + pass + return None + + last = _as_int(input_shape[-1]) + second = _as_int(input_shape[1]) + return last in (1, 3) and second is not None and second not in (1, 3) + + +def _resolve_input_dims(config: dict) -> tuple[int, int, int]: + """Resolve (width, height, channels) for the model input. + + Explicit `inputWidth`/`inputHeight` (threaded from the catalog's + `inputSize.{width,height}`) win; they fall back to the square `inputSize` + (the max side) for legacy square models. `inputChannels` defaults to 3 + (RGB); 1 selects the grayscale CTC recognizer path. Pure — no numpy/PIL. + """ + input_size = int(config.get("inputSize", 640)) + width = int(config.get("inputWidth") or input_size) + height = int(config.get("inputHeight") or input_size) + channels = int(config.get("inputChannels") or 3) + return width, height, channels + + +def _to_model_tensor(arr: "np.ndarray", channels_last: bool) -> "np.ndarray": + """Build the batched RGB model-input tensor from an HWC float array in [0,1]. + + - ``channels_last`` (ArcFace/CLIP NHWC) → ``[1,H,W,C]`` untransposed. + - otherwise → standard detector NCHW ``[1,C,H,W]``. + + Grayscale CTC recognizers do NOT use this — they need aspect-preserving + resize + edge padding, see {@link _preprocess_ctc_gray}. Pure numpy. + """ + if channels_last: + return arr[np.newaxis].astype(np.float32) + return arr.transpose(2, 0, 1)[np.newaxis].astype(np.float32) + + +_IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32) +_IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32) + +# InsightFace's SCRFD input contract: upstream feeds `(pixel - 127.5) / 128` +# ([-0.996, 0.996]), not the pool's historical plain `/255` ([0,1]). Expressed +# against the pool's already-`/255` HWC array (`arr`, not raw pixels): +# (arr*255 - 127.5) / 128 == (arr - _SCRFD_MEAN) / _SCRFD_STD +# with mean/std pre-divided by 255, the same shape of transform as the +# ImageNet branch below. See `postprocessors/scrfd.py` for the upstream +# citation and the measured effect (188 → 196 GT faces found, +# 2026-09-26 model-replacement spike, §2.4-2). +_SCRFD_MEAN = 127.5 / 255.0 +_SCRFD_STD = 128.0 / 255.0 + + +def _apply_normalization(arr: "np.ndarray", config: dict) -> "np.ndarray": + """Apply per-model input normalization to an HWC RGB float array in [0,1]. + + ``inputNormalization == 'imagenet'`` subtracts the ImageNet per-channel + mean and divides by the std (RGB), matching the EfficientNet-Lite0 / + MobileNetV3 animal + vehicle classifiers whose ``labels.json`` declares + ``normalize: imagenet``. ``'scrfd'`` applies InsightFace's SCRFD contract + (`_SCRFD_MEAN`/`_SCRFD_STD` above) — the `scrfd-2.5g` catalog entry. + Absent / ``'none'`` / ``'zero-one'`` keep the plain ``/255`` array + byte-identical to the historical unconditional path (most detectors, + CLIP/ArcFace, and the AIY bird classifier which bakes its own scale). Runs + on the HWC array BEFORE the optional NHWC/NCHW transpose in + :func:`_to_model_tensor`, so it is layout-agnostic (channel axis is always + last here). Pure numpy. + """ + mode = config.get("inputNormalization") + if mode not in ("imagenet", "scrfd"): + return arr + if arr.ndim != 3 or arr.shape[-1] != 3: + return arr + if mode == "scrfd": + return (arr - _SCRFD_MEAN) / _SCRFD_STD + return (arr - _IMAGENET_MEAN) / _IMAGENET_STD + + +def _preprocess_ctc_gray(img: "Image.Image", input_w: int, input_h: int) -> "np.ndarray": + """EasyOCR-accurate preprocess for a fixed-width grayscale CTC recognizer. + + Mirrors EasyOCR's AlignCollate + NormalizePAD exactly: + 1. grayscale by luminance (PIL ``convert('L')``), + 2. resize to height ``input_h`` PRESERVING aspect ratio (width capped at + ``input_w``) with BICUBIC — the recognizer was trained on undistorted + glyphs; stretching the crop to the full ``input_w`` warps glyph aspect + ~4x and the model reads noise (low-confidence garbage), + 3. normalize to ``[-1,1]`` (``(x/255 - 0.5)/0.5``), + 4. right-pad the width up to ``input_w`` by replicating the last column + (EasyOCR NormalizePAD edge padding — NOT zeros). + + Returns a ``[1,1,input_h,input_w]`` float32 tensor. Pure numpy/PIL. + """ + gray = img.convert("L") + ow, oh = gray.size + ratio = (ow / oh) if oh > 0 else 1.0 + resized_w = int(np.ceil(input_h * ratio)) + resized_w = max(1, min(input_w, resized_w)) + gray = gray.resize((resized_w, input_h), Image.BICUBIC) + arr = np.asarray(gray, dtype=np.float32) / 255.0 # (H, resized_w) + arr = (arr - 0.5) / 0.5 # [-1, 1] + if resized_w < input_w: + edge = arr[:, resized_w - 1 : resized_w] + pad = np.repeat(edge, input_w - resized_w, axis=1) + arr = np.concatenate([arr, pad], axis=1) + return arr[np.newaxis, np.newaxis].astype(np.float32) # (1, 1, H, W) + + +# --------------------------------------------------------------------------- +# Fix 2 — OpenVINO PrePostProcessor (preprocess folded into the compiled graph) +# --------------------------------------------------------------------------- + +# Always ON for OpenVINO; a module constant only so tests can exercise the +# plain-path compile. A failed fold already falls back per model (above). +_OV_PPP_ENABLED: bool = True + + +def _ov_dim_to_int(dim: Any) -> "int | None": + """Best-effort static-length extraction from an OpenVINO Dimension (or a + plain int). Dynamic / unknown dims return None. Pure — no numpy.""" + try: + return int(dim) + except (TypeError, ValueError): + pass + try: + if getattr(dim, "is_static", False): + return int(dim.get_length()) + except (TypeError, ValueError, AttributeError): + pass + return None + + +def _ov_ppp_plan( + dims: "list[int | None] | None", + input_channels: int, + preprocess_mode: str, + normalization: "str | None", + enabled: bool, +) -> "dict | None": + """Pure decision: should preprocessing be folded into the OpenVINO graph? + + `dims` is the model's declared input partial-shape as a list of int|None in + NCHW order. We only fold for standard 3-channel NCHW detectors/classifiers + with a STATIC spatial size — grayscale CTC recognizers (channels==1) and + NHWC-native embedders (ArcFace/CLIP, channel dim last) keep the pure-Python + path. Returns a `ppp_meta` dict describing the Python-side uint8 producer, + or None when PPP is not applicable. No OpenVINO / numpy imports — + unit-testable in isolation.""" + if not enabled: + return None + if input_channels == 1: + return None + if dims is None or len(dims) != 4: + return None + _n, c, h, w = dims + # NCHW with a concrete 3-channel dim. A last-dim of 3 (NHWC-native) leaves + # c as a spatial value → rejected here, so embedders stay on the float path. + if c != 3: + return None + if h is None or w is None or h <= 0 or w <= 0: + return None + return { + "inputW": int(w), + "inputH": int(h), + "size": int(max(w, h)), + "letterbox": preprocess_mode == "letterbox", + "normalization": normalization or "none", + } + + +def _preprocess_ov_uint8( + img: "Image.Image", plan: dict, +) -> tuple["np.ndarray", float, tuple[int, int]]: + """Produce the uint8 NHWC ``[1,H,W,3]`` input for a PPP-folded OV model. + + Letterbox (aspect-preserving pad) or plain resize stays in cheap uint8 PIL + space; the compiled OV graph does the float convert + /255 + optional + mean/std + NHWC→NCHW transpose on the inference device. Returns + ``(tensor, scale, pad)`` — the scale/pad still drive the postprocessor's box + mapping exactly like the letterbox float path. Pure numpy/PIL.""" + if plan["letterbox"]: + canvas, scale_val, pad = letterbox_image(img, plan["size"]) + else: + w, h = plan["inputW"], plan["inputH"] + if img.size == (w, h): + canvas, scale_val, pad = img, 1.0, (0, 0) + else: + canvas, scale_val, pad = img.resize((w, h), Image.BILINEAR), 1.0, (0, 0) + pix = np.asarray(canvas, dtype=np.uint8) + if pix.ndim == 2: + pix = np.stack([pix] * 3, axis=-1) + return pix[np.newaxis], scale_val, pad + + +def _build_ov_ppp_model(core: Any, path: str, config: dict) -> "tuple[Any, dict] | tuple[None, None]": + """Read the OpenVINO IR and, when eligible, attach a PrePostProcessor that + accepts a uint8 NHWC tensor and folds convert→scale(→mean/std)→layout into + the compiled graph. Returns ``(model, ppp_meta)`` to compile, or + ``(None, None)`` when PPP is not applied (caller compiles the path directly, + byte-identical to the historical float path). Raises on hard OV errors so + the caller's try/except falls back safely.""" + from openvino import Layout, Type + from openvino.preprocess import PrePostProcessor + + model = core.read_model(path) + inp = model.input() + try: + pshape = inp.get_partial_shape() + dims = [_ov_dim_to_int(pshape[i]) for i in range(len(pshape))] + except Exception: + dims = None + + _iw, _ih, input_channels = _resolve_input_dims(config) + plan = _ov_ppp_plan( + dims, + input_channels, + config.get("preprocessMode", "letterbox"), + config.get("inputNormalization"), + _OV_PPP_ENABLED, + ) + if plan is None: + return None, None + + ppp = PrePostProcessor(model) + # Tensor as the caller feeds it: uint8, NHWC, already at the model's spatial + # size (Python letterboxes/resizes to size in uint8) → OV needs no resize. + ppp.input().tensor().set_element_type(Type.u8).set_layout(Layout("NHWC")) + ppp.input().model().set_layout(Layout("NCHW")) + steps = ppp.input().preprocess().convert_element_type(Type.f32) + if plan["normalization"] == "imagenet": + # ((x/255) - mean) / std, applied in add-order (scale divides, mean + # subtracts). RGB channel order matches the fed PIL tensor. + steps.scale([255.0, 255.0, 255.0]) + steps.mean([float(v) for v in _IMAGENET_MEAN]) + steps.scale([float(v) for v in _IMAGENET_STD]) + elif plan["normalization"] == "scrfd": + # ((x/255) - _SCRFD_MEAN) / _SCRFD_STD — InsightFace's SCRFD contract, + # folded the same way as the ImageNet branch above (see + # `_apply_normalization`/`_SCRFD_MEAN`/`_SCRFD_STD`). + steps.scale([255.0, 255.0, 255.0]) + steps.mean([_SCRFD_MEAN, _SCRFD_MEAN, _SCRFD_MEAN]) + steps.scale([_SCRFD_STD, _SCRFD_STD, _SCRFD_STD]) + else: + steps.scale([255.0, 255.0, 255.0]) + built = ppp.build() + return built, plan + + +def _preprocess(img: Image.Image, config: dict) -> tuple[dict, float, tuple[int, int]]: + input_size = config.get("inputSize", 640) + input_w, input_h, input_channels = _resolve_input_dims(config) + # Bench frames have _bench_frame_id tag → use preprocess cache + bench_fid = getattr(img, '_bench_frame_id', None) + if bench_fid is not None: + global _bench_cache_hits, _bench_cache_misses + cache_key = (bench_fid, input_size) + cached = _bench_preprocess_cache.get(cache_key) + if cached is not None: + _bench_cache_hits += 1 + return cached + _bench_cache_misses += 1 + preprocess_mode = config.get("preprocessMode", "letterbox") + input_dict: dict = {} + if _runtime == "coreml": + input_name = config.get("_input_name", "image") + input_type = config.get("_input_type", "imageType") + input_shape = config.get("_input_shape") + if input_type == "imageType": + # FAST PATH — stay in uint8 PIL space. CoreML's predict() + # accepts a PIL Image directly and handles the + # BGR/normalize/CVPixelBuffer dance internally on its own + # (zero-copy when the size matches the model input). Going + # via numpy float32 → /255 → ×255 → astype(uint8) → Image. + # fromarray() costs ~16ms per 640×640 frame on M3 Pro and is + # PURE WASTE for imageType inputs. + if preprocess_mode == "letterbox": + canvas, scale_val, pad = letterbox_image(img, input_size) + elif img.size == (input_size, input_size): + canvas, scale_val, pad = img, 1.0, (0, 0) + else: + canvas, scale_val, pad = img.resize((input_size, input_size), Image.BILINEAR), 1.0, (0, 0) + input_dict[input_name] = canvas + arr = None # not used downstream for imageType — postprocess runs on predictions + elif input_shape is not None and len(input_shape) == 4: + if input_channels == 1: + # Grayscale CTC recognizer — EasyOCR aspect-preserving preprocess. + input_arr = _preprocess_ctc_gray(img, input_w, input_h) + scale_val = 1.0 + pad = (0, 0) + else: + if preprocess_mode == "letterbox": + arr, scale_val, pad = letterbox(img, input_size) + else: + arr = resize_image(img, input_w, input_h) + scale_val = 1.0 + pad = (0, 0) + # Per-model input normalization (imagenet mean/std for the + # animal + vehicle classifiers; no-op for everything else). + arr = _apply_normalization(arr, config) + # NHWC vs NCHW from the model's declared last dim (1/3 ⇒ + # channels-last), exactly as before for 3-channel models. + _, _, _, w_or_c = input_shape + input_arr = _to_model_tensor(arr, w_or_c in (1, 3)) + input_dict[input_name] = input_arr + else: + input_arr = arr.transpose(2, 0, 1)[np.newaxis].astype(np.float32) + input_dict[input_name] = input_arr + + input_names = config.get("_input_names", []) + if "iouThreshold" in input_names: + input_dict["iouThreshold"] = 0.45 + if "confidenceThreshold" in input_names: + input_dict["confidenceThreshold"] = config.get("confidence", 0.25) + elif _runtime == "edgetpu": + # EdgeTPU models are FULL-INTEGER quantized: feed raw uint8 (or int8) + # pixels in NHWC [1,H,W,C] — NO float /255 normalization (that is baked + # into the model's quantization params). + input_name = config.get("_input_name", "images") + in_w = int(config.get("inputWidth") or config.get("inputSize", 300)) + in_h = int(config.get("inputHeight") or config.get("inputSize", 300)) + if preprocess_mode == "letterbox": + canvas, scale_val, pad = letterbox_image(img, max(in_w, in_h)) + rgb = canvas + else: + rgb = img if img.size == (in_w, in_h) else img.resize((in_w, in_h), Image.BILINEAR) + scale_val, pad = 1.0, (0, 0) + pix = np.asarray(rgb, dtype=np.uint8) + if pix.ndim == 2: + pix = np.stack([pix] * 3, axis=-1) + if config.get("_edgetpu_in_dtype") == "int8": + input_dict[input_name] = (pix.astype(np.int16) - 128).astype(np.int8)[np.newaxis] + else: + input_dict[input_name] = pix[np.newaxis] # uint8 NHWC + else: + # OpenVINO / ONNX — always need a float tensor. + input_name = config.get("_input_name", "images") + # Fix 2 — OpenVINO PrePostProcessor fast path. When the model was + # compiled with a folded uint8→float/scale/layout preprocessor + # (`_ov_ppp` plan present), skip ALL numpy float work here: produce a + # uint8 NHWC tensor (aspect-preserving letterbox stays in cheap PIL + # space) and let the compiled OV graph do convert/scale/transpose on + # the inference device. Removes the per-frame /255 + transpose that + # dominated the non-inference CPU cost in production. + ov_ppp = config.get("_ov_ppp") if _runtime == "openvino" else None + if ov_ppp is not None and input_channels != 1: + input_arr, scale_val, pad = _preprocess_ov_uint8(img, ov_ppp) + input_dict[input_name] = input_arr + result = (input_dict, scale_val, pad) + if bench_fid is not None: + _bench_preprocess_cache[(bench_fid, input_size)] = result + return result + # uint8-input ONNX models: super-gradients / Frigate YOLO-NAS exports + # bake the /255 preprocessing INTO the graph and declare a uint8 input + # tensor. Feeding the historical float32 [0,1] tensor is a hard + # onnxruntime type error, so feed raw uint8 pixels in the declared + # layout instead (`_input_dtype` is captured at session load). + if config.get("_input_dtype") == "uint8" and input_channels == 3: + in_side = max(input_w, input_h) + if preprocess_mode == "letterbox": + canvas, scale_val, pad = letterbox_image(img, in_side) + else: + canvas = ( + img + if img.size == (input_w, input_h) + else img.resize((input_w, input_h), Image.BILINEAR) + ) + scale_val, pad = 1.0, (0, 0) + pix = np.asarray(canvas, dtype=np.uint8) + if pix.ndim == 2: + pix = np.stack([pix] * 3, axis=-1) + if _is_channels_last(config.get("_input_shape")): + input_dict[input_name] = pix[np.newaxis] + else: + input_dict[input_name] = pix.transpose(2, 0, 1)[np.newaxis] + result = (input_dict, scale_val, pad) + if bench_fid is not None: + _bench_preprocess_cache[(bench_fid, input_size)] = result + return result + if input_channels == 1: + # Grayscale CTC recognizer (EasyOCR plate-OCR): aspect-preserving + # resize + edge-pad + [-1,1] norm → [1,1,H,W]. NOT the squared + # 3-channel RGB tensor (which the runtime rejected — the reason + # plate recognition never ran). + input_dict[input_name] = _preprocess_ctc_gray(img, input_w, input_h) + scale_val = 1.0 + pad = (0, 0) + else: + if preprocess_mode == "letterbox": + arr, scale_val, pad = letterbox(img, input_size) + else: + arr = resize_image(img, input_w, input_h) + scale_val = 1.0 + pad = (0, 0) + # Per-model input normalization (imagenet mean/std for the animal + + # vehicle classifiers; no-op for detectors, CLIP, bird, etc.). + arr = _apply_normalization(arr, config) + # Feed the layout the model actually declares. NHWC models (ArcFace + # face-embedding [N,112,112,3], CLIP) must NOT be NCHW-transposed — + # a strict backend (NPU) throws on the wrong shape and the embedding + # is never produced (the multi-day face-collection outage). Unknown + # shape → NCHW (the prior behaviour), so detectors are unaffected. + input_dict[input_name] = _to_model_tensor( + arr, _is_channels_last(config.get("_input_shape")) + ) + + result = (input_dict, scale_val, pad) + if bench_fid is not None: + _bench_preprocess_cache[(bench_fid, input_size)] = result + return result + + +def _postprocess( + predictions: dict, + slot_config: dict, + orig_w: int, + orig_h: int, + scale_val: float, + pad: tuple[int, int], + elapsed_ms: float, + preprocess_ms: float = 0.0, + predict_ms: float = 0.0, + batch_size: int = 1, +) -> dict: + postprocessor_type = slot_config.get("postprocessor", "yolo") + postprocessor_fn = POSTPROCESSORS.get(postprocessor_type) + if postprocessor_fn is None: + raise ValueError(f"Unknown postprocessor: {postprocessor_type}") + result = postprocessor_fn(predictions, slot_config, orig_w, orig_h, scale_val, pad) + result["inferenceMs"] = round(elapsed_ms, 2) + result["preprocessMs"] = round(preprocess_ms, 2) + result["predictMs"] = round(predict_ms, 2) + result["batchSize"] = batch_size + result["frameSize"] = f"{orig_w}x{orig_h}" + return result + + +# --------------------------------------------------------------------------- +# Runtime dispatcher — pipelined prepare → predict → postprocess +# --------------------------------------------------------------------------- + + +class RuntimeDispatcher: + """Split-stage dispatcher that lets preprocess run ahead of predict. + + Stages use separate thread pools so an in-flight `predict` doesn't + block the preprocess for the next frame: + + camera A: preprocess → predict ─┐ + camera B: preprocess → predict ─┐ + camera C: preprocess → predict ─┐ + + While predict for A runs on the predict_pool, preprocess for B can + run on the prepare_pool; each stage releases the Python event loop + between calls so the reader stays responsive. + + `concurrency` sets the prepare pool size AND the predict pool size. + For CoreML/OpenVINO the runtime itself serialises access to the + device, so a single predict worker is usually enough. ONNX Runtime + benefits from >1 (its InferenceSession.run is thread-safe and the + native thread pool schedules concurrent runs). + """ + + def __init__(self, runtime: str, concurrency: int) -> None: + workers = max(1, concurrency) + self._workers = workers + self._prepare_pool = concurrent.futures.ThreadPoolExecutor( + max_workers=workers, + thread_name_prefix=f"{runtime}-prep", + ) + self._predict_pool = concurrent.futures.ThreadPoolExecutor( + max_workers=workers, + thread_name_prefix=f"{runtime}-predict", + ) + + @property + def workers(self) -> int: + return self._workers + + async def run(self, slot: ModelSlot, img: Image.Image) -> dict: + loop = asyncio.get_event_loop() + t_start = time.perf_counter() + orig_w, orig_h = img.size + # Stage 1 — preprocess. Runs on prepare_pool; overlaps with + # any predict already scheduled for a previous frame. + input_dict, scale_val, pad = await loop.run_in_executor( + self._prepare_pool, _preprocess, img, slot.config, + ) + t_predict_start = time.perf_counter() + preprocess_ms = (t_predict_start - t_start) * 1000 + # Stage 2 — predict. Dedicated pool so preprocessors for later + # frames can run while the runtime works on this one. + predictions = await loop.run_in_executor( + self._predict_pool, slot.predict_fn, input_dict, + ) + # Stage 3 — postprocess. Cheap CPU-bound step; reuse the prepare + # pool to avoid tying up a predict worker on scalar math. + t_predict_end = time.perf_counter() + predict_ms = (t_predict_end - t_predict_start) * 1000 + elapsed_ms = (t_predict_end - t_start) * 1000 + return await loop.run_in_executor( + self._prepare_pool, + _postprocess, + predictions, slot.config, orig_w, orig_h, scale_val, pad, elapsed_ms, + preprocess_ms, predict_ms, 1, + ) + + async def run_cached( + self, + slot: ModelSlot, + img: Image.Image, + cache_key: tuple[int, int], + prep_cache: dict[tuple[int, int], tuple[dict, float, tuple[int, int], int, int]], + ) -> dict: + """Like run() but caches the preprocessed input dict. Second+ + calls for the same (frame_id, model_idx) skip preprocess entirely + — goes straight to predict + postprocess. Saves ~10ms/call.""" + loop = asyncio.get_event_loop() + t_start = time.perf_counter() + orig_w, orig_h = img.size + + cached = prep_cache.get(cache_key) + if cached is not None: + input_dict, scale_val, pad, _, _ = cached + preprocess_ms = 0.0 + else: + input_dict, scale_val, pad = await loop.run_in_executor( + self._prepare_pool, _preprocess, img, slot.config, + ) + preprocess_ms = (time.perf_counter() - t_start) * 1000 + prep_cache[cache_key] = (input_dict, scale_val, pad, orig_w, orig_h) + + t_predict_start = time.perf_counter() + predictions = await loop.run_in_executor( + self._predict_pool, slot.predict_fn, input_dict, + ) + t_predict_end = time.perf_counter() + predict_ms = (t_predict_end - t_predict_start) * 1000 + elapsed_ms = (t_predict_end - t_start) * 1000 + return await loop.run_in_executor( + self._prepare_pool, + _postprocess, + predictions, slot.config, orig_w, orig_h, scale_val, pad, elapsed_ms, + preprocess_ms, predict_ms, 1, + ) + + async def run_list(self, slot: ModelSlot, imgs: list[Image.Image]) -> list[dict]: + """Batch predict path — single CoreML/OV/ORT predict call processing + all N items, then split + per-item postprocess. + + For CoreML: passes a list of input dicts to `model.predict(...)`, + which CoreML iterates internally with shared session state — saves + per-call ANE setup overhead vs a thread-pool fan-out. + + For OpenVINO/ONNX: falls back to a loop on the same predict thread + (no model-level batching since their predict_fn signatures take one + array). Still saves the asyncio.gather + ThreadPoolExecutor dispatch + overhead. + """ + if not imgs: + return [] + loop = asyncio.get_event_loop() + t_start = time.perf_counter() + orig_sizes = [img.size for img in imgs] + batch_size = len(imgs) + + # Stage 1 — preprocess all in parallel on the prepare pool. + prepared = await asyncio.gather(*[ + loop.run_in_executor(self._prepare_pool, _preprocess, img, slot.config) + for img in imgs + ]) + # prepared = list of (input_dict, scale_val, pad) + t_predict_start = time.perf_counter() + preprocess_ms = (t_predict_start - t_start) * 1000 + + # Stage 2 — single predict call on the predict thread. + runtime = _runtime + supports_batch = bool(slot.config.get("_supports_batch")) + max_batch = int(slot.config.get("_max_batch", 1)) + if runtime == "coreml" and supports_batch and batch_size <= max_batch: + # FAST PATH — stack the N preprocessed (1,3,H,W) tensors + # into a single (N,3,H,W) tensor and issue ONE predict call. + # The model was exported with a flexible batch axis so this + # produces a single ANE dispatch processing all N frames in + # parallel — amortises CoreML session setup AND the per-call + # compute_units context-switch. + input_name = slot.config.get("_input_name", "image") + stacked = np.concatenate([p[0][input_name] for p in prepared], axis=0) + + def stacked_predict() -> list[dict]: + names = slot.input_names + payload: dict = {input_name: stacked} + # Some models accept side-channel inputs (iouThreshold + # etc.). Forward only those that the current model + # accepts; the stacked input is set above. + first = prepared[0][0] + for k, v in first.items(): + if k != input_name and (not names or k in names): + payload[k] = v + out = slot.model.predict(payload) + # Split each output along axis 0 back into N items. + # Output shape (N, ...); slice per index and rebuild + # per-item dicts so postprocess sees the same shape it + # would for a single-frame predict. + results: list[dict] = [] + for i in range(batch_size): + item: dict = {} + for k, v in out.items(): + if hasattr(v, "shape") and len(v.shape) > 0 and v.shape[0] == batch_size: + item[k] = v[i:i + 1] + else: + item[k] = v + results.append(item) + return results + + predictions_list = await loop.run_in_executor(self._predict_pool, stacked_predict) + elif runtime == "coreml": + # Fallback — model.predict([list of input dicts]) iterates + # internally on shared session state. Not as fast as stacked + # but still saves the asyncio.gather + ThreadPoolExecutor + # dispatch overhead vs N separate calls. + def batched_predict() -> list[dict]: + names = slot.input_names + if names: + cleaned = [{k: v for k, v in p[0].items() if k in names} for p in prepared] + else: + cleaned = [p[0] for p in prepared] + return slot.model.predict(cleaned) + + predictions_list = await loop.run_in_executor(self._predict_pool, batched_predict) + else: + # OpenVINO / ONNX — loop on the predict thread to keep IPC + # amortisation (one executor dispatch instead of N) without + # requiring model-level batched inputs. + def looped_predict() -> list[dict]: + return [slot.predict_fn(p[0]) for p in prepared] + + predictions_list = await loop.run_in_executor(self._predict_pool, looped_predict) + + # Stage 3 — postprocess each result. Per-item elapsed shares the + # batch wall (fair attribution) — callers see it as `inferenceMs`. + t_predict_end = time.perf_counter() + predict_ms = (t_predict_end - t_predict_start) * 1000 + elapsed_ms = (t_predict_end - t_start) * 1000 + return await asyncio.gather(*[ + loop.run_in_executor( + self._prepare_pool, + _postprocess, + predictions, slot.config, + orig_sizes[i][0], orig_sizes[i][1], + prepared[i][1], prepared[i][2], + elapsed_ms, + preprocess_ms, predict_ms, batch_size, + ) + for i, predictions in enumerate(predictions_list) + ]) + + def close(self) -> None: + self._prepare_pool.shutdown(wait=False) + self._predict_pool.shutdown(wait=False) + + +# --------------------------------------------------------------------------- +# Per-model backpressure — bound in-flight inference, drop-oldest shedding +# --------------------------------------------------------------------------- + +# Frames a single model may hold WAITING behind its running set before the +# OLDEST waiting frame is shed. Running slots (= predict-pool size) keep the +# device fed; a couple of pending frames cover dispatch gaps. Anything beyond +# that only sits in an invisible executor queue inflating per-frame +# "inference" time — on an overloaded node (N100 iGPU) that queue wait +# ballooned to 9-22s per frame. Shedding it instead makes overload read as +# `dropped` frames (fast `"dropped": true` responses) rather than latency. +MAX_PENDING_PER_MODEL: int = 2 + + +@dataclass +class _ModelQueueState: + """Mutable per-model counters for ModelBackpressure.""" + running: int = 0 + pending: "deque[Any]" = field(default_factory=deque) + + +class ModelBackpressure: + """Per-model in-flight bound with drop-oldest shedding. + + Pure decision core — no asyncio, no I/O — so it is unit-testable + (see test_inference_pool_backpressure.py). The event loop calls: + + admit(model_idx, item) on every single-frame inference arrival. + Returns (to_run, to_drop): items to dispatch NOW and items shed + (oldest queued first). At most one of the lists is non-empty. + complete(model_idx) when a dispatched inference finishes. + Returns the next queued items to dispatch (0 or 1). + + Why per-model and why these bounds: the predict pool has `workers` + threads shared by all models; a model gets at most `max_running` + concurrently-dispatched inferences (sized to the predict pool), plus + a `max_pending` waiting buffer. The previous GLOBAL ceiling + (`workers * 4`, min 16) was unreachable from the live pipeline — the + TS runner's semaphore keeps at most `maxConcurrentInferences` frames + in flight per node, well below 16 — so overload never shed frames; it + just queued inside the executor as multi-second "inference" time. + """ + + def __init__(self, max_running: int, max_pending: int) -> None: + self._max_running = max(1, max_running) + self._max_pending = max(0, max_pending) + self._states: dict[int, _ModelQueueState] = {} + + def _state(self, model_idx: int) -> _ModelQueueState: + state = self._states.get(model_idx) + if state is None: + state = _ModelQueueState() + self._states[model_idx] = state + return state + + def admit(self, model_idx: int, item: Any) -> tuple[list, list]: + """Admit one inference request; returns (to_run, to_drop).""" + state = self._state(model_idx) + if state.running < self._max_running: + state.running += 1 + return [item], [] + state.pending.append(item) + dropped: list = [] + while len(state.pending) > self._max_pending: + # DROP-OLDEST: the newest frame is the most valuable one for a + # live detection pipeline — shed the stalest queued frame(s). + dropped.append(state.pending.popleft()) + return [], dropped + + def complete(self, model_idx: int) -> list: + """One running inference finished; returns queued items to start.""" + state = self._state(model_idx) + state.running = max(0, state.running - 1) + if state.pending and state.running < self._max_running: + state.running += 1 + return [state.pending.popleft()] + return [] + + def snapshot(self) -> dict: + """Per-model {running, pending} — queue depth for the mem_stats line. + + A leak shows RSS climbing while these stay flat; a genuinely growing + working set shows them climbing together. Pure read, no mutation. + """ + return { + str(idx): {"running": st.running, "pending": len(st.pending)} + for idx, st in self._states.items() + } + + +# --------------------------------------------------------------------------- +# Memory diagnostics (mem_stats command) +# +# The pools leak over hours and get OOM-killed (2026-08-18: anon-rss 3.8 GB / +# 8.8 GB, twice in six hours) — the Node-side watchdog samples RSS from +# /proc//status, and asks THIS process once per sweep for the internals +# /proc cannot see: what the pool has been asked to do (request counters), +# what it is holding (caches, queue depths) and what the allocator is doing +# (arena count — the lead on total-vm 47.5 GB vs 11.3 GB at similar RSS). +# Everything here is a cheap O(1)-ish read; nothing walks the heap. +# --------------------------------------------------------------------------- + + +@dataclass +class PoolCounters: + """Requests served since spawn, by opcode. Monotonic, wraps never.""" + + infer_jpeg: int = 0 + infer_raw: int = 0 + infer_batch: int = 0 + infer_cached: int = 0 + commands: int = 0 + frames_cached: int = 0 + # Requests answered `dropped` because their wire deadline had already + # passed when they would have run — dead work NOT executed (D350). For a + # batch, every item counts. + shed_deadline_expired: int = 0 + + def as_dict(self) -> dict: + return { + "inferJpeg": self.infer_jpeg, + "inferRaw": self.infer_raw, + "inferBatch": self.infer_batch, + "inferCached": self.infer_cached, + "commands": self.commands, + "framesCached": self.frames_cached, + "shedDeadlineExpired": self.shed_deadline_expired, + "totalInference": ( + self.infer_jpeg + self.infer_raw + self.infer_batch + self.infer_cached + ), + } + + +def _read_self_memory() -> dict: + """RSS / VmSize / peak-RSS / swap / thread count for THIS process. + + /proc on Linux (one 4 KB read); getrusage maxrss everywhere as backstop. + Partial results are fine — an empty dict means the read failed entirely. + """ + out: dict = {} + try: + import resource + + ru = resource.getrusage(resource.RUSAGE_SELF) + # ru_maxrss is kilobytes on Linux, BYTES on macOS. + out["maxRssKb"] = ( + int(ru.ru_maxrss // 1024) if sys.platform == "darwin" else int(ru.ru_maxrss) + ) + except Exception: + pass + try: + with open("/proc/self/status", "r") as fh: + text = fh.read() + for label, key in ( + ("VmRSS", "rssKb"), + ("VmSize", "vmKb"), + ("VmHWM", "hwmKb"), + ("VmSwap", "swapKb"), + ): + match = re.search(rf"^{label}:\s+(\d+)\s*kB", text, re.M) + if match: + out[key] = int(match.group(1)) + match = re.search(r"^Threads:\s+(\d+)", text, re.M) + if match: + out["threads"] = int(match.group(1)) + except Exception: + pass + return out + + +def _malloc_arena_count() -> Optional[int]: + """Count glibc malloc arenas via malloc_info(3) into an in-memory stream. + + Each arena reserves up to 64 MB of VIRTUAL space — the count is the direct + test of the "total-vm 47.5 GB is allocator arenas" hypothesis from the + 2026-08-18 OOM audit. glibc/Linux only; None elsewhere or on any failure + (never raises — this rides the mem_stats reply and must not kill it). + """ + if not sys.platform.startswith("linux"): + return None + try: + import ctypes + + libc = ctypes.CDLL(None, use_errno=True) + libc.open_memstream.restype = ctypes.c_void_p + libc.open_memstream.argtypes = [ctypes.c_void_p, ctypes.c_void_p] + libc.malloc_info.argtypes = [ctypes.c_int, ctypes.c_void_p] + libc.fclose.argtypes = [ctypes.c_void_p] + libc.free.argtypes = [ctypes.c_void_p] + + buf = ctypes.c_char_p() + size = ctypes.c_size_t() + stream = libc.open_memstream(ctypes.byref(buf), ctypes.byref(size)) + if not stream: + return None + rc = libc.malloc_info(0, stream) + libc.fclose(stream) # flushes and finalises buf/size + try: + if rc != 0 or not buf.value: + return None + xml = ctypes.string_at(buf, size.value) + return xml.count(b" dict: + """Assemble the mem_stats reply. Pure given its inputs except the two + process-wide reads (memory + arenas), so the shape is unit-testable.""" + return { + "cmd": "mem_stats", + "status": "ok", + "memory": _read_self_memory(), + "mallocArenas": _malloc_arena_count(), + "gcCounts": list(gc.get_count()), + "counters": counters.as_dict(), + "backpressure": backpressure.snapshot(), + "modelsLoaded": sum(1 for s in models if getattr(s, "loaded", False)), + "frameCacheEntries": frame_cache_len, + "preprocessCacheEntries": preprocess_cache_len, + "benchPreprocessCacheEntries": bench_cache_len, + "uptimeSec": uptime_sec, + # What is compiling, and whether a compile ever failed to return + # (D653) — the one fact a host needs to tell "slow" from "poisoned". + "modelLoad": model_load, + } + + +# --------------------------------------------------------------------------- +# Single-frame inference execution — module-level so the deadline gate is +# unit-testable (test_inference_pool_deadline.py) without driving the whole +# event loop. `handle_inference` in _run() delegates here; the caller's +# `finally` still owns the backpressure slot release. +# --------------------------------------------------------------------------- + + +async def _execute_inference( + req_id: int, + img: Any, + model_idx: int, + deadline_ms: int, + *, + models: "list[ModelSlot]", + dispatcher: Any, + writer: Any, + batch_mode: str, + get_accumulator: "Optional[Callable[[int], Any]]", + counters: "PoolCounters", +) -> None: + """Deadline gate + predict for ONE admitted single-frame request. + + The deadline is checked HERE — immediately before predict — rather than + only at arrival, because this coroutine is entered both on admission and + when the backpressure queue promotes a waiting frame: the wait is exactly + where a request goes stale. The gate comes first (before the model-loaded + check) on purpose — the cheapest question first, and an abandoned request + deserves a shed, not a diagnosis. `dispatcher` and `writer` are + duck-typed (RuntimeDispatcher / ResponseWriter in production). + """ + now_ms = time.time() * 1000.0 + if _deadline_expired(deadline_ms, now_ms): + counters.shed_deadline_expired += 1 + _log_deadline_shed(counters.shed_deadline_expired, model_idx, now_ms - deadline_ms) + await writer.send(req_id, _dropped_deadline_payload()) + return + if model_idx >= len(models) or not models[model_idx].loaded: + await writer.send(req_id, { + "error": f"Model {model_idx} not loaded", + "kind": "detections", + "detections": [], + "inferenceMs": 0, + }) + return + if batch_mode == "window" and get_accumulator is not None: + await get_accumulator(model_idx).submit(req_id, img) + return + result = await dispatcher.run(models[model_idx], img) + await writer.send(req_id, result) + + +# --------------------------------------------------------------------------- +# IPC — binary framing with request_id multiplexing +# --------------------------------------------------------------------------- + + +HEADER_LEN = 4 # total_len prefix +PREFIX_LEN = HEADER_LEN + 4 + 1 # total_len + req_id + msg_type + + +async def _read_exact(reader: asyncio.StreamReader, n: int) -> bytes: + return await reader.readexactly(n) + + +class ResponseWriter: + """Serialises writes to the stdout pipe; safe from multiple coroutines.""" + + def __init__(self, writer: asyncio.StreamWriter) -> None: + self._writer = writer + self._lock = asyncio.Lock() + + async def send(self, req_id: int, payload: dict) -> None: + # numpy scalars (float32 / int64 / bool_) are NOT JSON-serializable + # under the stdlib encoder. With numpy 2.x the previous lenient + # path is gone, so we must coerce before encoding. Cheap default + # callback only kicks in for unhandled types — Python natives skip + # it entirely. + data = json.dumps(payload, default=_json_default).encode("utf-8") + total_len = 4 + len(data) # req_id(4) + json + header = struct.pack(" Any: + """Coerce numpy scalars/arrays to JSON-friendly Python natives.""" + if isinstance(obj, np.generic): + return obj.item() + if isinstance(obj, np.ndarray): + return obj.tolist() + raise TypeError(f"Object of type {obj.__class__.__name__} is not JSON serializable") + + +# --------------------------------------------------------------------------- +# Command handlers +# --------------------------------------------------------------------------- + +# Config keys a LOADED slot can absorb through `reconfigure` — decode-scoped +# values the postprocessor re-reads from `slot.config` on every inference, so +# they take effect on the next frame with no session reload. Mirrors +# `PoolDecodeSettings` on the TypeScript side; adding a key here without adding +# it there (or vice versa) reintroduces the D214 dead-knob class. +RECONFIGURABLE_CONFIG_KEYS = frozenset({"nmsIouThreshold"}) + + +def _handle_command(models: list[ModelSlot], cmd: dict) -> dict: + """The model commands that need no compile: unload, reconfigure, status. + + `load` and `replace` are NOT here — they compile, and a compile must never + run on the event loop (D653). They go through `_handle_model_command`, + which the `CommandRouter` awaits off the loop's reader. + """ + action = cmd.get("cmd") + if action == "unload": + index = cmd["index"] + if index < len(models) and models[index].loaded: + _unload_model(models[index]) + sys.stderr.write(f"Model {index} unloaded\n") + sys.stderr.flush() + return {"cmd": "unload", "index": index, "status": "ok"} + + if action == "reconfigure": + # Retune a LOADED slot's decode-scoped settings in place — no session + # re-instantiation. `slot.config` is re-read by the postprocessor on + # every inference (see `_postprocess`), so mutating these keys is + # picked up by the next frame without a reload. + # + # ONLY the allow-listed keys are accepted. A `path` or `inputSize` + # arriving here would be written into the config while the loaded + # session kept serving the OLD model — a silent, undetectable lie. + # Anything else is refused loudly; the caller must use `replace`. + index = cmd["index"] + patch = cmd.get("config") or {} + unknown = sorted(k for k in patch if k not in RECONFIGURABLE_CONFIG_KEYS) + if unknown: + return { + "cmd": "reconfigure", + "index": index, + "status": "error", + "error": ( + f"keys not reconfigurable without a reload: {', '.join(unknown)} " + f"(allowed: {', '.join(sorted(RECONFIGURABLE_CONFIG_KEYS))})" + ), + } + if index >= len(models) or not models[index].loaded: + return { + "cmd": "reconfigure", + "index": index, + "status": "error", + "error": f"model {index} is not loaded", + } + slot = models[index] + before = {k: slot.config.get(k) for k in patch} + slot.config.update(patch) + sys.stderr.write(f"Model {index} reconfigured: {before} -> {dict(patch)}\n") + sys.stderr.flush() + return {"cmd": "reconfigure", "index": index, "status": "ok"} + + if action == "status": + status = [] + for i, slot in enumerate(models): + status.append({ + "index": i, + "path": slot.config.get("path") if slot.loaded else None, + "loaded": slot.loaded, + "postprocessor": slot.config.get("postprocessor") if slot.loaded else None, + }) + return {"cmd": "status", "models": status, "benchCache": _bench_cache_stats()} + + return {"cmd": action or "unknown", "status": "error", "error": f"Unknown command: {action}"} + + +# --------------------------------------------------------------------------- +# Model loads run OFF the event loop, one at a time, under a hard bound (D653) +# --------------------------------------------------------------------------- +# +# 2026-09-26, hub `openvino:gpu`: one `core.compile_model()` of +# `camstack-yunet-2023mar.xml` for the iGPU never returned. It ran INLINE on this +# loop, so from 18:33:48 until a container restart at 22:41 the worker read no +# request and wrote no reply: 786 client-side timeouts on six cameras, 123 of +# 123 `mem_stats` probes lost, and a process that looked perfectly alive. +# +# So a load now runs on its own DAEMON thread and the loop only awaits it: +# inference and `mem_stats` keep being served while a compile runs. And it is +# BOUNDED: past `MODEL_LOAD_TIMEOUT_SEC` the load answers `compile-timeout`, +# naming the model, instead of never answering. +# +# A native thread stuck in a GPU compiler cannot be cancelled or killed from +# Python. So a timeout POISONS the worker FOR NEW LOADS: every later load is +# refused at once (`worker-poisoned`) rather than queued behind a thread that +# may never free the compiler, and the models already loaded keep serving. +# +# The abandoned compile is NOT abandoned work (fix round 1): it keeps running, +# so a slow but finite compile still writes its cache. When it comes back the +# worker says so on the event channel (`compile-finished-late`) and is usable +# for loads again — the next load of that model is a warm cache hit. If it +# never comes back, the host kills the process at its HARD bound +# (shared-inference-pool.ts `POOL_MODEL_LOAD_BOUNDS`). Daemon, not an executor +# thread: `concurrent.futures` joins its threads at interpreter exit, so a stuck +# compile would also have made the worker impossible to shut down. + +# SOFT bound on ONE model load — compile included — in seconds. A cold GPU +# compile of the whole model set on the N100 iGPU measured ~25 s; one model is +# normally 1-3 s. The host declares its own value in the startup config +# (`modelLoadTimeoutMs`) so its reply deadline and this bound cannot drift; +# this is what an old host that declares nothing gets. +MODEL_LOAD_TIMEOUT_SEC = 120.0 + +# Request id reserved for UNSOLICITED worker events (a reply to no request). +# The host never allocates it (shared-inference-pool.ts `WORKER_EVENT_REQ_ID`). +WORKER_EVENT_REQ_ID = 0xFFFFFFFF + +# Commands answered by the loop immediately, never queued behind a load: the +# probe the host uses to tell "slow" from "wedged" must never wait for the very +# compile it is asking about. +INLINE_COMMANDS = frozenset({"mem_stats", "uncache_frame"}) + +# Commands that compile a model — the only ones that leave the loop. +LOAD_COMMANDS = frozenset({"load", "replace"}) + + +def _model_load_timeout_sec(config: dict) -> float: + """The load bound for this connection: the host's, else the default.""" + raw = config.get("modelLoadTimeoutMs") + if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw > 0: + return float(raw) / 1000.0 + return MODEL_LOAD_TIMEOUT_SEC + + +def _model_id_of(config: dict) -> str: + """The model's file stem — what a log line and the host's reply carry.""" + path = str(config.get("path") or "") + stem = os.path.splitext(os.path.basename(path))[0] + return stem or "unknown" + + +class ModelLoadTimeout(Exception): + """A load did not finish inside its bound; new loads are refused until it does.""" + + +class WorkerPoisoned(Exception): + """A previous load is still running past its bound; no new load starts.""" + + +class ModelLoader: + """Runs `_load_model` on a daemon thread under a hard bound. + + One load at a time is the CALLER's guarantee (the `CommandRouter` consumes + model commands in order); this class only owns the thread, the bound and + the poisoning. `load_fn` is injectable for tests; production passes + nothing and gets the module's `_load_model`, looked up at call time. + `notify` sends an unsolicited event to the host (the `compile-finished-late` + line); absent, the event is only written to stderr. + """ + + def __init__( + self, + timeout_sec: float = MODEL_LOAD_TIMEOUT_SEC, + load_fn: "Optional[Callable[[ModelSlot, dict], None]]" = None, + notify: "Optional[Callable[[dict], Awaitable[None]]]" = None, + ) -> None: + self.timeout_sec = timeout_sec + self._load_fn = load_fn + self._notify = notify + self.finished_late = 0 + # Test seam: runs on the load thread right after it has claimed the + # outcome — lets a test hold the thread exactly in the race window. + self._after_claim: Optional[Callable[[], None]] = None + # Monotonic start of the compile running past its bound, if any — the + # host counts its HARD bound from here, not from a later refusal. + self._abandoned_started: Optional[float] = None + # `ensure_future` tasks are only weakly referenced by the loop; keep + # each notification alive until it is sent. + self._pending_notifications: "set[asyncio.Future[None]]" = set() + self.in_flight: Optional[dict] = None + self.poisoned_by: Optional[dict] = None + self.completed = 0 + self.failed = 0 + self.timed_out = 0 + + def snapshot(self) -> dict: + """What the `mem_stats` reply carries about loads.""" + in_flight = None + if self.in_flight is not None: + in_flight = { + "index": self.in_flight["index"], + "modelId": self.in_flight["modelId"], + "elapsedMs": round((time.monotonic() - self.in_flight["startedAt"]) * 1000), + } + return { + "inFlight": in_flight, + "poisonedBy": dict(self.poisoned_by) if self.poisoned_by is not None else None, + "boundMs": round(self.timeout_sec * 1000), + "completed": self.completed, + "failed": self.failed, + "timedOut": self.timed_out, + "finishedLate": self.finished_late, + } + + def _finish_abandoned( + self, index: int, model_id: str, outcome: Optional[BaseException], elapsed_ms: int, + ) -> None: + """On the loop: an abandoned compile came back. The compiler is free + again, so the worker loads again; a successful one has written its + cache, so the host's next load of the model is warm. Its RESULT is + still discarded — the host was told it failed and has moved on.""" + self.finished_late += 1 + self.poisoned_by = None + self._abandoned_started = None + payload = { + "event": "compile-finished-late", + "index": index, + "modelId": model_id, + "ok": outcome is None, + "elapsedMs": elapsed_ms, + } + if outcome is not None: + payload["error"] = str(outcome) + if self._notify is not None: + task = asyncio.ensure_future(self._notify(payload)) + self._pending_notifications.add(task) + task.add_done_callback(self._pending_notifications.discard) + + def abandoned_elapsed_ms(self) -> Optional[int]: + """How long the compile running past its bound has been running.""" + if self._abandoned_started is None: + return None + return round((time.monotonic() - self._abandoned_started) * 1000) + + async def load(self, slot: ModelSlot, config: dict, index: int) -> None: + """Load `config` into the DETACHED `slot`, or raise. + + Raises `WorkerPoisoned` without starting anything once a previous load + timed out, `ModelLoadTimeout` when this one exceeds the bound, and the + load's own exception when it fails in time. + """ + model_id = _model_id_of(config) + if self.poisoned_by is not None: + raise WorkerPoisoned( + f"worker poisoned for loads: the load of {self.poisoned_by['modelId']} is still " + f"running past its bound ({self.poisoned_by['boundMs']}ms) — no new load starts " + f"until it returns" + ) + load_fn = self._load_fn if self._load_fn is not None else _load_model + loop = asyncio.get_running_loop() + done: "asyncio.Future[None]" = loop.create_future() + # Exactly ONE of "the load returned in time" and "the bound fired" + # wins, decided under this lock (release review). Without it the + # thread could see "not abandoned", the bound fire before its settle + # ran, and the worker stay poisoned forever for a load that finished. + decision = threading.Lock() + claimed: dict = {"by": None} # "thread" | "bound" + + def settle(exc: Optional[BaseException]) -> None: + if done.done(): + return + if exc is None: + done.set_result(None) + else: + done.set_exception(exc) + + def target() -> None: + outcome: Optional[BaseException] = None + try: + load_fn(slot, config) + except BaseException as exc: # noqa: BLE001 — handed to the awaiting loop + outcome = exc + with decision: + late = claimed["by"] == "bound" + if not late: + claimed["by"] = "thread" + if self._after_claim is not None: + self._after_claim() + if late: + # The bound already fired. Say that the native call DID come + # back, and when — the only evidence of how long a slow + # compile really takes — and hand the worker back its loads. + elapsed_ms = round((time.monotonic() - started) * 1000) + sys.stderr.write( + f"Model {index} load of {model_id} returned AFTER its bound " + f"in {elapsed_ms}ms ({'error: ' + str(outcome) if outcome else 'ok'}) " + f"— result discarded, loads re-enabled\n" + ) + sys.stderr.flush() + try: + loop.call_soon_threadsafe( + self._finish_abandoned, index, model_id, outcome, elapsed_ms, + ) + except RuntimeError: + pass # loop closed: the process is shutting down + return + try: + loop.call_soon_threadsafe(settle, outcome) + except RuntimeError: + pass # loop closed: the process is shutting down + + started = time.monotonic() + self.in_flight = {"index": index, "modelId": model_id, "startedAt": started} + thread = threading.Thread(target=target, name=f"model-load-{index}", daemon=True) + thread.start() + try: + try: + await asyncio.wait_for(asyncio.shield(done), timeout=self.timeout_sec) + except asyncio.TimeoutError: + with decision: + bound_won = claimed["by"] is None + if bound_won: + claimed["by"] = "bound" + if bound_won: + raise + # The load returned AT the bound: its settle is already on + # its way to this loop. It counts as in time. + await done + except asyncio.TimeoutError: + self.timed_out += 1 + self._abandoned_started = started + self.poisoned_by = { + "index": index, + "modelId": model_id, + "boundMs": round(self.timeout_sec * 1000), + "at": round(time.time() * 1000), + } + raise ModelLoadTimeout( + f"load of {model_id} (index {index}) did not return within " + f"{round(self.timeout_sec * 1000)}ms — it keeps running so its cache " + f"can be written; no new load starts until it returns" + ) from None + except Exception: + self.failed += 1 + raise + finally: + self.in_flight = None + self.completed += 1 + + +async def _handle_model_command( + models: "list[ModelSlot]", cmd: dict, loader: ModelLoader, +) -> dict: + """Every model-management command, in order; `load`/`replace` off the loop. + + The new model is loaded into a DETACHED slot and installed only once its + load completed inside the bound, so a compile that returns late — after the + host was told `compile-timeout` — can never surface as a model the host + believes failed. Whatever the index held before keeps serving until that + swap: a `replace` that fails or times out leaves the old model in place, + and the old slot object is only dropped, never mutated, so a predict + already holding it finishes normally. + """ + action = cmd.get("cmd") + if action not in LOAD_COMMANDS: + return _handle_command(models, cmd) + index = cmd["index"] + config = cmd["config"] + model_id = _model_id_of(config) + while len(models) <= index: + models.append(ModelSlot()) + slot = ModelSlot() + sys.stderr.write( + f"Model {index} load started: {config['path']} " + f"(bound {round(loader.timeout_sec * 1000)}ms)\n" + ) + sys.stderr.flush() + t0 = time.perf_counter() + try: + await loader.load(slot, config, index) + except WorkerPoisoned as exc: + return { + "cmd": action, "index": index, "status": "error", + "reason": "worker-poisoned", "modelId": model_id, "poisoned": True, + # How long the EARLIER compile has run: the host's hard bound + # counts from its start, not from this refusal. + "compileElapsedMs": loader.abandoned_elapsed_ms(), + "error": str(exc), + } + except ModelLoadTimeout as exc: + sys.stderr.write(f"Model {index} load TIMED OUT: {exc}\n") + sys.stderr.flush() + return { + "cmd": action, "index": index, "status": "error", + "reason": "compile-timeout", "modelId": model_id, "poisoned": True, + "timeoutMs": round(loader.timeout_sec * 1000), + "compileElapsedMs": loader.abandoned_elapsed_ms(), + "error": str(exc), + } + except Exception as exc: + reply = {"cmd": action, "index": index, "status": "error", "error": str(exc)} + # A typed refusal (e.g. `native-load-crashed`) names its class, so the + # host can tell it from an ordinary failed load. + reason = getattr(exc, "reason", None) + if isinstance(reason, str): + reply["reason"] = reason + reply["modelId"] = model_id + return reply + models[index] = slot + load_ms = round((time.perf_counter() - t0) * 1000) + verb = "loaded" if action == "load" else "replaced" + sys.stderr.write(f"Model {index} {verb}: {config['path']} ({load_ms}ms)\n") + sys.stderr.flush() + return {"cmd": action, "index": index, "status": "ok", "loadMs": load_ms} + + +class CommandRouter: + """Routes MSG_COMMAND frames without ever blocking the reader. + + `INLINE_COMMANDS` are answered at once by `inline` (it needs the loop's own + state: caches, backpressure, counters). Everything else is queued and + consumed IN ORDER by one task — model slots are indices shared with the + host, so a `status` or `unload` must never overtake the `load` sent before + it. + """ + + def __init__( + self, + models: "list[ModelSlot]", + loader: ModelLoader, + send: "Callable[[int, dict], Awaitable[None]]", + inline: "Callable[[dict], dict]", + ) -> None: + self._models = models + self._loader = loader + self._send = send + self._inline = inline + self._queue: "asyncio.Queue[tuple[int, dict]]" = asyncio.Queue() + self._consumer: "Optional[asyncio.Task[None]]" = None + + async def submit(self, req_id: int, cmd: dict) -> None: + if cmd.get("cmd") in INLINE_COMMANDS: + try: + response = self._inline(cmd) + except Exception as exc: + response = {"cmd": cmd.get("cmd") or "unknown", "status": "error", "error": str(exc)} + await self._send(req_id, response) + return + self._queue.put_nowait((req_id, cmd)) + if self._consumer is None or self._consumer.done(): + self._consumer = asyncio.create_task(self._consume()) + + async def _consume(self) -> None: + while not self._queue.empty(): + req_id, cmd = self._queue.get_nowait() + try: + response = await _handle_model_command(self._models, cmd, self._loader) + except Exception as exc: + response = {"cmd": cmd.get("cmd") or "unknown", "status": "error", "error": str(exc)} + await self._send(req_id, response) + + +# --------------------------------------------------------------------------- +# Main async loop +# --------------------------------------------------------------------------- + + +async def _run() -> None: + loop = asyncio.get_event_loop() + reader = asyncio.StreamReader(loop=loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=loop) + await loop.connect_read_pipe(lambda: protocol, sys.stdin.buffer) + + transport, stdout_protocol = await loop.connect_write_pipe( + asyncio.streams.FlowControlMixin, sys.stdout.buffer, + ) + stream_writer = asyncio.StreamWriter(transport, stdout_protocol, None, loop) + writer = ResponseWriter(stream_writer) + + t_start = time.perf_counter() + + # ── Startup — first message is config (req_id=0, msg_type=COMMAND) ── + header = await _read_exact(reader, PREFIX_LEN) + total_len, req_id = struct.unpack(" concurrency: + sys.stderr.write( + f"OpenVINO LATENCY: predict concurrency {concurrency} -> {target}\n" + ) + sys.stderr.flush() + concurrency = target + + dispatcher = RuntimeDispatcher(runtime, concurrency) + # Backpressure: bound in-flight single-frame inference PER MODEL, shedding + # the OLDEST queued frame when full (see ModelBackpressure). max_running is + # sized to the predict pool — dispatching more than `workers` concurrent + # inferences for one model can't run any faster, it only queues invisibly + # inside the executor and balloons per-frame "inference" time (9-22s + # observed on the N100 iGPU). Healthy nodes never fill the pending buffer, + # so this is a no-op except under genuine overload. + backpressure = ModelBackpressure( + max_running=dispatcher.workers, + max_pending=MAX_PENDING_PER_MODEL, + ) + # Served-request counters + spawn time for the mem_stats command — the + # denominator that lets the watchdog tell "RSS grew because it worked" + # from "RSS grew per unit of work" (a leak). + counters = PoolCounters() + pool_started_monotonic = time.monotonic() + startup_ms = round((time.perf_counter() - t_start) * 1000) + loaded_count = sum(1 for s in models if s.loaded) + sys.stderr.write( + f"Ready — {loaded_count}/{len(models)} model(s) in {startup_ms}ms " + f"(runtime={runtime}, workers={dispatcher.workers})\n" + ) + sys.stderr.flush() + + await writer.send(req_id, { + "status": "ready", + "models": loaded_count, + "total": len(models), + "startupMs": startup_ms, + "runtime": runtime, + "workers": dispatcher.workers, + # Our own capability, NOT the negotiated minimum: the Node side runs + # the same min() against what it declared, so both ends derive the + # identical effective version (D350). + "protocolVersion": PROTOCOL_VERSION, + }) + + # ── Window accumulator (per-model) ────────────────────────────── + # + # In `window` mode, every MSG_INFER_RAW arrival pushes (req_id, img) + # into a per-model pending queue and arms a `window_ms` timer. When + # the timer fires, OR the queue reaches `max_batch_size`, we flush + # via `dispatcher.run_list(slot, imgs)` — single batched predict. + # Concurrent inferRaw calls from N cameras coalesce into one CoreML + # predict call, amortising ANE setup. + + class WindowAccumulator: + def __init__(self, model_idx: int) -> None: + self.model_idx = model_idx + self.pending: list[tuple[int, Image.Image]] = [] + self.flush_task: Optional[asyncio.Task[None]] = None + + async def submit(self, req_id: int, img: Image.Image) -> None: + self.pending.append((req_id, img)) + if len(self.pending) >= max_batch_size: + # Reached cap — cancel any pending timer (we are flushing + # immediately from the caller's coroutine, not the timer) + # and flush now. + if self.flush_task is not None: + self.flush_task.cancel() + self.flush_task = None + await self._flush() + return + if self.flush_task is None: + self.flush_task = asyncio.create_task(self._delayed_flush()) + + async def _delayed_flush(self) -> None: + try: + await asyncio.sleep(window_ms / 1000.0) + except asyncio.CancelledError: + return + # Mark the timer task as complete BEFORE the dispatch await so + # `_flush` does not try to cancel the running task (which would + # raise CancelledError at `await dispatcher.run_list` and drop + # responses on the floor — the symptom the TS side observed as + # `runPipeline.mutate` hanging forever). + self.flush_task = None + await self._flush() + + async def _flush(self) -> None: + batch = self.pending + self.pending = [] + if not batch: + return + slot = models[self.model_idx] + try: + imgs = [b[1] for b in batch] + results = await dispatcher.run_list(slot, imgs) + except Exception as exc: + sys.stderr.write(f"Window flush error (model {self.model_idx}): {exc}\n") + sys.stderr.flush() + err_payload = { + "error": str(exc), + "kind": "detections", + "detections": [], + "inferenceMs": 0, + } + for rid, _ in batch: + await writer.send(rid, err_payload) + return + for (rid, _), result in zip(batch, results): + await writer.send(rid, result) + + accumulators: dict[int, WindowAccumulator] = {} + + # ── Bench frame cache ────────────────────────────────────────── + # Stores raw PIL Images keyed by uint32 frame_id. The Node side + # sends MSG_CACHE_FRAME once with the full 1.2MB payload; all + # subsequent MSG_INFER_CACHED calls send only a 5-byte header + # (model_idx + frame_id). Eliminates the 35ms/call pipe transfer + # that dominates bench throughput. + frame_cache: dict[int, Image.Image] = {} + + # Preprocessed-input cache: keyed by (frame_id, model_idx). First + # inferCached call preprocesses and caches; subsequent calls skip + # preprocess entirely and go straight to predict + postprocess. + # Saves ~10ms/call (the PIL resize + numpy + tensor pack cost). + _preprocess_cache: dict[tuple[int, int], tuple[dict, float, tuple[int, int], int, int]] = {} + + def inline_command(cmd: dict) -> dict: + # Needs the loop's own state (caches, backpressure, counters), which + # the model-command path never sees. Answered at once — this is the + # host's liveness probe, and it must not wait for a compile. + if cmd.get("cmd") == "mem_stats": + return _mem_stats_payload( + models, + counters, + backpressure, + len(frame_cache), + len(_preprocess_cache), + len(_bench_preprocess_cache), + round(time.monotonic() - pool_started_monotonic), + model_load=model_loader.snapshot(), + ) + fid = cmd.get("frameId", -1) + frame_cache.pop(fid, None) + # Purge preprocessed tensor caches for this frame + for k in [k for k in _preprocess_cache if k[0] == fid]: + del _preprocess_cache[k] + # Also purge the bench preprocess cache + for k in [k for k in _bench_preprocess_cache if k[0] == fid]: + del _bench_preprocess_cache[k] + return {"cmd": "uncache_frame", "status": "ok", "frameId": fid} + + command_router = CommandRouter(models, model_loader, writer.send, inline_command) + + def get_accumulator(model_idx: int) -> WindowAccumulator: + acc = accumulators.get(model_idx) + if acc is None: + acc = WindowAccumulator(model_idx) + accumulators[model_idx] = acc + return acc + + # ── Main loop ─────────────────────────────────────────────────── + async def handle_inference( + req_id: int, img: Image.Image, model_idx: int, deadline_ms: int = 0, + ) -> None: + try: + # The deadline gate lives in _execute_inference, immediately + # before predict — it runs here on admission AND again when a + # queued frame is promoted below, which is where a request goes + # stale (D350). + await _execute_inference( + req_id, img, model_idx, deadline_ms, + models=models, + dispatcher=dispatcher, + writer=writer, + batch_mode=batch_mode, + get_accumulator=get_accumulator, + counters=counters, + ) + except Exception as exc: + sys.stderr.write(f"Inference error (model {model_idx}, req {req_id}): {exc}\n") + sys.stderr.flush() + await writer.send(req_id, { + "error": str(exc), + "kind": "detections", + "detections": [], + "inferenceMs": 0, + }) + finally: + # Release this model's running slot; promote the next queued + # frame (if any) into the freed slot immediately. + for next_req_id, next_img, next_deadline_ms in backpressure.complete(model_idx): + asyncio.create_task( + handle_inference(next_req_id, next_img, model_idx, next_deadline_ms) + ) + + def _send_dropped(req_id: int) -> None: + # Fast shed response — returned in microseconds instead of queuing the + # frame for seconds. The TS side (shared-inference-pool.ts) recognises + # `dropped: true` and counts it, so overload is measurable, not silent. + asyncio.create_task(writer.send(req_id, { + "kind": "detections", + "detections": [], + "inferenceMs": 0, + "dropped": True, + })) + + def _dispatch_inference( + req_id: int, img: Image.Image, model_idx: int, deadline_ms: int = 0, + ) -> None: + # Gate every single-frame inference through the per-model in-flight + # bound. Slot accounting is synchronous (admit here, complete in the + # handle_inference `finally`), so a slot can never leak past a task's + # lifetime. Under overload the OLDEST queued frame is shed and answered + # immediately so the caller never hangs on a frame we chose not to run. + # The wire deadline rides the queued tuple so the gate in + # _execute_inference sees it AFTER the wait, not just at arrival. + to_run, to_drop = backpressure.admit(model_idx, (req_id, img, deadline_ms)) + for dropped_req_id, _dropped_img, _dropped_deadline in to_drop: + _send_dropped(dropped_req_id) + for run_req_id, run_img, run_deadline_ms in to_run: + asyncio.create_task(handle_inference(run_req_id, run_img, model_idx, run_deadline_ms)) + + async def handle_batch( + req_id: int, model_idx: int, items: list[Image.Image], deadline_ms: int = 0, + ) -> None: + # Deadline gate first — the whole batch is one abandoned caller, so + # every item is answered as a well-formed dropped record and none of + # the work runs (D350). + now_ms = time.time() * 1000.0 + if _deadline_expired(deadline_ms, now_ms): + counters.shed_deadline_expired += len(items) + _log_deadline_shed(counters.shed_deadline_expired, model_idx, now_ms - deadline_ms) + await writer.send(req_id, { + "results": [_dropped_deadline_payload() for _ in items], + }) + return + if model_idx >= len(models) or not models[model_idx].loaded: + await writer.send(req_id, { + "error": f"Model {model_idx} not loaded", + "results": [], + }) + return + try: + slot = models[model_idx] + if batch_mode in ("list", "window"): + # `list` mode: send the batch through dispatcher.run_list + # — single predict call, internal CoreML iteration. + # `window` mode: same dispatch path (no point queueing into + # the accumulator when caller already packed N items). + payload = await dispatcher.run_list(slot, items) + else: + tasks = [dispatcher.run(slot, img) for img in items] + results = await asyncio.gather(*tasks, return_exceptions=True) + payload = [] + for r in results: + if isinstance(r, Exception): + payload.append({ + "error": str(r), + "kind": "detections", + "detections": [], + "inferenceMs": 0, + }) + else: + payload.append(r) + await writer.send(req_id, {"results": payload}) + except Exception as exc: + sys.stderr.write(f"Batch error (model {model_idx}, req {req_id}): {exc}\n") + sys.stderr.flush() + await writer.send(req_id, {"error": str(exc), "results": []}) + + while True: + try: + header = await _read_exact(reader, PREFIX_LEN) + except asyncio.IncompleteReadError: + return # stdin closed + total_len, req_id = struct.unpack(" 0 else b"" + except asyncio.IncompleteReadError: + return + + # v2 sheddable payloads carry [8B absolute deadline] first; v1 (and + # every command) passes through untouched with deadline 0 (D350). + try: + deadline_ms, payload = _split_deadline(msg_type, payload, wire_version) + except ValueError as exc: + await writer.send(req_id, {"error": str(exc)}) + continue + + if msg_type == MSG_COMMAND: + counters.commands += 1 + try: + cmd = json.loads(payload) + except Exception as exc: + await writer.send(req_id, {"cmd": "unknown", "status": "error", "error": str(exc)}) + continue + # NEVER awaits a model load: `load`/`replace` are queued and + # compiled on their own thread under a bound (D653), so the next + # frame is read while a GPU compile runs. + await command_router.submit(req_id, cmd) + + elif msg_type == MSG_INFER_JPEG: + counters.infer_jpeg += 1 + if len(payload) < 1: + await writer.send(req_id, {"error": "empty infer_jpeg payload"}) + continue + model_idx = payload[0] + jpeg = payload[1:] + try: + img = decode_jpeg(jpeg) + except Exception as exc: + await writer.send(req_id, {"error": f"jpeg decode failed: {exc}"}) + continue + _dispatch_inference(req_id, img, model_idx, deadline_ms) + + elif msg_type == MSG_INFER_RAW: + counters.infer_raw += 1 + if len(payload) < 10: + await writer.send(req_id, {"error": "truncated infer_raw header"}) + continue + model_idx = payload[0] + width, height = struct.unpack(" len(payload): + parse_err = "truncated batch item header" + break + width, height = struct.unpack(" len(payload): + parse_err = "truncated batch item payload" + break + raw = payload[offset:offset + size] + offset += size + try: + item_img = wrap_raw(raw, width, height, fmt) + if batch_frame_id != 0: + item_img._bench_frame_id = batch_frame_id + items.append(item_img) + except Exception as exc: + parse_err = f"raw wrap failed: {exc}" + break + if parse_err is not None: + await writer.send(req_id, {"error": parse_err, "results": []}) + continue + asyncio.create_task(handle_batch(req_id, model_idx, items, deadline_ms)) + + elif msg_type == MSG_CACHE_FRAME: + counters.frames_cached += 1 + # Wire: [4B frame_id][4B width][4B height][1B fmt][raw bytes] + if len(payload) < 13: + await writer.send(req_id, {"error": "truncated cache_frame header"}) + continue + fid = struct.unpack("= len(models) or not models[model_idx].loaded: + await writer.send(req_id, {"error": f"model {model_idx} not loaded"}) + continue + _dispatch_inference(req_id, img, model_idx, deadline_ms) + + else: + await writer.send(req_id, {"error": f"unknown msg_type: {msg_type}"}) + + +def main() -> None: + try: + asyncio.run(_run()) + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main() diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/__init__.py b/camstack_train/_vendor/pipeline_python/postprocessors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f1fe1e30b339687f14084a4f5f31fb580cd7b5c8 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/__init__.py @@ -0,0 +1,38 @@ +"""Postprocessor dispatch table for inference_pool.py. + +Each postprocessor takes raw model output (dict of numpy arrays) and model config, +returns a structured dict matching the StepOutput JSON format. +""" +from .yolo import postprocess_yolo +from .yolo_seg import postprocess_yolo_seg +from .scrfd import postprocess_scrfd +from .arcface import postprocess_arcface +from .clip import postprocess_clip +from .softmax import postprocess_softmax +from .ctc import postprocess_ctc +from .plate_slots import postprocess_plate_slots +from .saliency import postprocess_saliency +from .yamnet import postprocess_yamnet +from .ssd import postprocess_ssd +from .rfdetr import postprocess_rfdetr +from .yolonas import postprocess_yolonas +from .yunet import postprocess_yunet +from .yolo_split_dfl import postprocess_yolo_split_dfl + +POSTPROCESSORS = { + "yolo": postprocess_yolo, + "yolo-seg": postprocess_yolo_seg, + "scrfd": postprocess_scrfd, + "arcface": postprocess_arcface, + "clip": postprocess_clip, + "softmax": postprocess_softmax, + "ctc": postprocess_ctc, + "plate-slots": postprocess_plate_slots, + "saliency": postprocess_saliency, + "yamnet": postprocess_yamnet, + "ssd": postprocess_ssd, + "rfdetr": postprocess_rfdetr, + "yolonas": postprocess_yolonas, + "yunet": postprocess_yunet, + "yolo-split-dfl": postprocess_yolo_split_dfl, +} diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/arcface.py b/camstack_train/_vendor/pipeline_python/postprocessors/arcface.py new file mode 100644 index 0000000000000000000000000000000000000000..ead1e959bae3993999d2398ed736c41b56f1e68f --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/arcface.py @@ -0,0 +1,31 @@ +"""ArcFace face embedding postprocessor. + +Input: raw embedding tensor from ArcFace model +Output: {"kind": "embedding", "embedding": [...], "embeddingNorm": [...]} +""" +import numpy as np + + +def postprocess_arcface( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """L2-normalize the embedding vector.""" + # Get the first output tensor (usually named "fc1" or similar) + raw = np.array(list(predictions.values())[0]).flatten().astype(np.float32) + + norm = np.linalg.norm(raw) + if norm == 0: + normalized = np.zeros_like(raw) + else: + normalized = raw / norm + + return { + "kind": "embedding", + "embedding": raw.tolist(), + "embeddingNorm": normalized.tolist(), + } diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/clip.py b/camstack_train/_vendor/pipeline_python/postprocessors/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..10109f54e305d314338ed5271607523ed181dd54 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/clip.py @@ -0,0 +1,35 @@ +"""CLIP image embedding postprocessor. + +Input: raw embedding tensor from a CLIP vision encoder (e.g. MobileCLIP S0/S1/S2) +Output: {"kind": "embedding", "embedding": [...], "embeddingNorm": [...]} + +MobileCLIP uses resize 256×256 + rescale /255 only (no ImageNet mean/std). +The existing inference_pool resize_image path already produces this layout — +preprocessMode:'resize' + inputNormalization:'none' is the correct config. +""" +import numpy as np + + +def postprocess_clip( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """L2-normalize the CLIP embedding vector.""" + # Get the first output tensor (vision encoder emits a single embedding) + raw = np.array(list(predictions.values())[0]).flatten().astype(np.float32) + + norm = np.linalg.norm(raw) + if norm == 0: + normalized = np.zeros_like(raw) + else: + normalized = raw / norm + + return { + "kind": "embedding", + "embedding": raw.tolist(), + "embeddingNorm": normalized.tolist(), + } diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/ctc.py b/camstack_train/_vendor/pipeline_python/postprocessors/ctc.py new file mode 100644 index 0000000000000000000000000000000000000000..4050c88e5bff69c7d65aae38a8410242252f4cb8 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/ctc.py @@ -0,0 +1,406 @@ +"""CTC text recognition postprocessor. + +Used for: vgg_english_g2 (plate OCR). + +Input: [1, seqLen, numChars] logits +Output: {"kind": "text", "text": "ABC123", "confidence": 0.95, "formatValid": true} + +Algorithm: argmax per timestep → collapse consecutive duplicates → remove blank (index 0) → join + +Plausibility gate +----------------- +When the recognizer is fed a too-low-resolution / heavily-blurred plate crop +its logit map collapses toward the CTC blank and greedy decode emits a spurious +1–2 character read (e.g. a lone "A") at whatever confidence the single surviving +timestep carries. That noise was being stored verbatim as a "plate" label. The +charset + greedy decode themselves are correct — this only rejects a result too +short to be a real plate (`minTextLength`, non-space chars) or below a confidence +floor (`minTextConfidence`), returning empty text so no garbage label is stored. +Real plates read at 5+ chars / conf ~0.86–1.0, so the conservative defaults never +suppress a genuine read; both are config-overridable for per-camera tuning. + +Normalisation before the grammar +-------------------------------- +The recognizer's charset carries lowercase letters and a space, and live reads +use them freely ("dn Ro 3o9" is the same plate as "DNRO309"). The grammar and +the confusable table are both defined over UPPERCASE alphanumerics, so a raw +read is upper-cased and stripped of non-alphanumerics BEFORE either runs — +otherwise a perfectly readable plate fails the grammar for its case, and the +0↔O repair cannot even be attempted (lowercase 'o' is not a CONFUSABLES key). +The per-character scores and aligned timesteps are filtered alongside the +glyphs, so the rescorer keeps reading the softmax mass at the right timestep. + +The RAW text still feeds the symbol gate below: normalisation must never be +able to launder hallucinated punctuation into a conforming plate. A read that +carries a symbol glyph is rejected before the grammar is ever consulted, so +repair only ever runs on a read that already passed the plausibility gate. + +Format-aware rescoring (post-hoc) +--------------------------------- +After the plausibility gate, the NORMALISED read is validated against the active +region's plate grammar (`plateRegion`, default 'DE'; 'off' disables). If the +letter/digit sequence already matches, it is accepted as-is (`formatValid: true`). +If it does NOT match, candidate reads are generated by substituting classic OCR +confusables (0↔O, 1↔I, 8↔B, 5↔S, 2↔Z, 6↔G, 4↔A) position-by-position; each +candidate is scored by the mean per-character probability and the highest-scoring +GRAMMAR-CONFORMING candidate replaces the read (`formatValid: true`). If no +candidate matches, the original greedy read is kept verbatim (`formatValid: +false`) — validation NEVER drops a plausible read, it only fixes or annotates it. + +Per-character probability approximation +--------------------------------------- +The greedy CTC path collapses consecutive-duplicate argmax timesteps; for each +emitted output character we retain the single timestep `t` at which its run +began. A candidate's per-character probability is then approximated by the +softmax at that ONE aligned timestep — for an unchanged position the original +argmax probability, and for a substituted position `probs[t, altIndex]` (the +real softmax mass the confusable already carried at that timestep). This is an +approximation: it does not re-run a full CTC forward alignment for the candidate +string (which could re-collapse differently). For single-character confusable +swaps it is a faithful, cheap proxy — the timestep that produced the original +glyph is exactly where its confusable competes for probability. +""" +import itertools +import re + +import numpy as np + +# Shortest plausible plate (non-space chars). No real plate is 1–2 chars, so +# this only rejects the degenerate near-all-blank collapse; kept conservative to +# never drop a genuine short/vanity plate. +DEFAULT_MIN_TEXT_LENGTH = 3 +# Confidence floor (0.0 = disabled by default — character count is the reliable +# discriminator for the degenerate collapse; the "A"@1.0 case is caught by +# length, not confidence). Available for per-camera tuning via config. +DEFAULT_MIN_TEXT_CONFIDENCE = 0.0 +# Product-level default plate grammar region — supplied to this postprocessor by +# the TS step config (`plateRegion`, see step-definitions.ts). When the key is +# ABSENT from `config` the postprocessor falls back to 'off' (no rescoring) so a +# bare / pre-plumbing call is byte-identical to the historical decode behaviour; +# the DE default only takes effect once the config plumbing supplies it. +DEFAULT_PLATE_REGION = "DE" +# Fallback when `plateRegion` is absent from config (backward-compatible: no +# rescoring, no formatValid annotation). +ABSENT_PLATE_REGION = "off" +# Cap on the number of positions that may simultaneously carry a confusable +# alternative. Each substitutable position doubles the candidate count, so the +# worst case is 2**MAX_SUBST_POSITIONS candidates. Plates are ≤8 glyphs so this +# is rarely binding, but the cap guarantees a bounded search (≤256 candidates) +# even on a pathological long read: when more positions than this have a +# confusable, only the LOWEST-confidence positions (the most likely misreads) +# stay substitutable; the rest are locked to their greedy glyph. +MAX_SUBST_POSITIONS = 8 + +# Classic OCR confusable pairs, bidirectional. Maps a glyph to the alternates it +# is commonly mistaken for/against. Used ONLY to repair a read that fails the +# grammar — never to alter a read that already conforms. +CONFUSABLES: dict[str, tuple[str, ...]] = { + "0": ("O",), + "O": ("0",), + "1": ("I",), + "I": ("1",), + "8": ("B",), + "B": ("8",), + "5": ("S",), + "S": ("5",), + "2": ("Z",), + "Z": ("2",), + "6": ("G",), + "G": ("6",), + "4": ("A",), + "A": ("4",), +} + +# Region plate grammars, keyed by ISO-3166 alpha-2 code. Each pattern validates +# the RAW letter/digit sequence the OCR emits (no separators — the recognizer +# never outputs the dash/space of the physical plate). Ready for future +# countries ('IT', 'FR', …); 'off' (handled before lookup) disables rescoring. +# +# GERMANY ('DE'): district code (1–3 letters) + 1–2 letters + 1–4 digits + +# optional E (electric) / H (historic) suffix; ≤8 glyphs excluding separators. +# Separators are optional in the pattern for robustness, though the OCR emits +# none. The letter-block split (district vs. series) is inherently ambiguous on +# the raw string, but that does not matter — the regex validates the whole +# sequence, not the split. +PLATE_GRAMMARS: dict[str, str] = { + "DE": r"^[A-ZÄÖÜ]{1,3}[- ]?[A-Z]{1,2}[- ]?[0-9]{1,4}[EH]?$", +} + + +def _matches_grammar(text: str, pattern) -> bool: + return pattern.match(text) is not None + + +def _normalize_read( + chars: list[str], + char_scores: list[float], + char_timesteps: list[int], +) -> tuple[str, list[float], list[int]]: + """Upper-case + drop non-alphanumeric glyphs, keeping scores/timesteps aligned. + + The grammar patterns and CONFUSABLES are defined over uppercase alnums; the + charset the recognizer decodes from is not. Returning the filtered score and + timestep lists alongside the text is what lets the rescorer keep reading a + candidate glyph's softmax mass at the timestep that produced its position. + + A glyph whose upper-case form is not a single character (no such glyph in + this charset, but the fold is locale-dependent in general) is kept as-is + rather than silently changing the string length. + """ + out_chars: list[str] = [] + out_scores: list[float] = [] + out_timesteps: list[int] = [] + for ch, score, timestep in zip(chars, char_scores, char_timesteps): + if not ch.isalnum(): + continue + upper = ch.upper() + out_chars.append(upper if len(upper) == 1 else ch) + out_scores.append(score) + out_timesteps.append(timestep) + return "".join(out_chars), out_scores, out_timesteps + + +def _rescore_for_grammar( + text: str, + char_scores: list[float], + char_timesteps: list[int], + probs: np.ndarray, + charset: list[str], + pattern, +) -> tuple[str, bool]: + """Return (repaired_or_original_text, format_valid). + + If `text` already matches `pattern`, return it unchanged with True. + Otherwise enumerate confusable substitutions (bounded), score each candidate + by mean per-character probability, and return the highest-scoring candidate + that matches the grammar. If none match, return the original text with False. + """ + if _matches_grammar(text, pattern): + return text, True + + chars = list(text) + n = len(chars) + + # Reverse charset lookup (glyph → first index) so a substituted glyph's + # softmax mass can be read from the aligned timestep. Built once per call. + char_to_index: dict[str, int] = {} + for idx, glyph in enumerate(charset): + if glyph and glyph not in char_to_index: + char_to_index[glyph] = idx + + # Which positions carry a usable confusable alternative (both the glyph has a + # mapping AND the alternate is representable in this charset so its timestep + # probability can be looked up). + subst_positions: list[int] = [] + for i, ch in enumerate(chars): + alts = CONFUSABLES.get(ch) + if not alts: + continue + if any(alt in char_to_index for alt in alts): + subst_positions.append(i) + + if not subst_positions: + return text, False + + # Bound the search: keep only the MAX_SUBST_POSITIONS lowest-confidence + # positions substitutable (most likely misreads); lock the rest. + if len(subst_positions) > MAX_SUBST_POSITIONS: + subst_positions.sort(key=lambda i: char_scores[i]) + subst_positions = sorted(subst_positions[:MAX_SUBST_POSITIONS]) + + subst_set = set(subst_positions) + + # Per-position option list: original glyph, plus any representable confusables + # (only for substitutable positions). itertools.product enumerates the grid. + options: list[list[str]] = [] + for i, ch in enumerate(chars): + if i in subst_set: + alts = [a for a in CONFUSABLES.get(ch, ()) if a in char_to_index] + options.append([ch, *alts]) + else: + options.append([ch]) + + def _prob_at(i: int, glyph: str) -> float: + if glyph == chars[i]: + return char_scores[i] + idx = char_to_index.get(glyph) + if idx is None: + return 0.0 + return float(probs[char_timesteps[i], idx]) + + best_text: str | None = None + best_score = -1.0 + for combo in itertools.product(*options): + candidate = "".join(combo) + if candidate == text: + continue # already known non-matching (checked above) + if not _matches_grammar(candidate, pattern): + continue + score = sum(_prob_at(i, combo[i]) for i in range(n)) / n if n else 0.0 + if score > best_score: + best_score = score + best_text = candidate + + if best_text is not None: + return best_text, True + return text, False + + +# Glyphs the recognizer emits for a plate's physical registration seal. +SEAL_GLYPHS = frozenset(".:;,*-'`\"·") + + +def _strip_seal_glyph(text: str, region: str) -> "str | None": + """Return the normalised plate when `text` is a conforming read carrying ONE + interior seal glyph; ``None`` otherwise (caller applies the normal gate). + + Narrow by construction: exactly one seal-like glyph, never at either end, + nothing else non-alphanumeric left over, and the normalised remainder must + match the region grammar. A read that does not become a legal plate is not + rescued. + """ + pattern_src = PLATE_GRAMMARS.get(region) + if pattern_src is None: + return None + seals = [i for i, c in enumerate(text) if c in SEAL_GLYPHS] + if len(seals) != 1: + return None + pos = seals[0] + if pos == 0 or pos == len(text) - 1: + return None + stripped = text[:pos] + text[pos + 1 :] + if any(not (c.isalnum() or c.isspace()) for c in stripped): + return None + normalised = "".join(c for c in stripped.upper() if c.isalnum()) + if re.match(pattern_src, normalised) is None: + return None + return normalised + + +def postprocess_ctc( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """CTC greedy decode.""" + charset = config.get("charset", []) + + # Get first output tensor + raw = np.array(list(predictions.values())[0]).astype(np.float32) + + # Handle shapes: [1, T, C] or [T, C] + if raw.ndim == 3: + raw = raw[0] # remove batch + # raw is now [T, C] — logits (not probabilities) + seq_len, num_chars = raw.shape + + # Apply softmax to get probabilities + raw_max = np.max(raw, axis=1, keepdims=True) + exp_raw = np.exp(raw - raw_max) + probs = exp_raw / np.sum(exp_raw, axis=1, keepdims=True) + + # Argmax per timestep + best_indices = np.argmax(probs, axis=1) # [T] + best_scores = probs[np.arange(seq_len), best_indices] # [T] + + # Collapse consecutive duplicates (retain the timestep each run began at so + # the rescorer can read a confusable's softmax mass at that aligned step). + collapsed = [] + collapsed_scores = [] + collapsed_timesteps = [] + for i, idx in enumerate(best_indices): + if i == 0 or idx != best_indices[i - 1]: + collapsed.append(int(idx)) + collapsed_scores.append(float(best_scores[i])) + collapsed_timesteps.append(i) + + # Remove blank (index 0) and join + chars = [] + scores = [] + char_timesteps = [] + for idx, score, t in zip(collapsed, collapsed_scores, collapsed_timesteps): + if idx != 0: + if idx < len(charset): + chars.append(charset[idx]) + else: + chars.append(chr(idx)) + scores.append(score) + char_timesteps.append(t) + + text = "".join(chars) + confidence = float(np.mean(scores)) if scores else 0.0 + + # Plausibility gate — suppress the degenerate near-all-blank collapse so + # low-resolution noise is never stored as a plate label. Count ALPHANUMERIC + # characters only ("DW RO 309" → 7): the charset carries symbol glyphs + # ('[', ')', '.', …) that a blurred crop hallucinates, and they must not + # count toward "long enough to be a plate". + min_len = int(config.get("minTextLength", DEFAULT_MIN_TEXT_LENGTH)) + min_conf = float(config.get("minTextConfidence", DEFAULT_MIN_TEXT_CONFIDENCE)) + region = str(config.get("plateRegion", ABSENT_PLATE_REGION)) + alnum_count = sum(1 for c in text if c.isalnum()) + # Symbol-blind gate: when a plate grammar/region is ACTIVE, no real plate + # contains a non-alphanumeric glyph (spaces allowed — word gaps are real). + # Observed live labels "[miv4o)" / "6ou.2" (2026-07-22) sailed through the + # length-only gate; reject them exactly like the other implausible reads. + has_symbol_glyph = region != "off" and any( + not (c.isalnum() or c.isspace()) for c in text + ) + # …except the SEAL. A German plate physically carries a registration seal + # between the district code and the letter group, and the recognizer + # transcribes that badge as a punctuation glyph ('.', ':', '-', …). Once the + # quad warp (2026-08-20) started delivering legible strips, the correct + # decodes arrived as "DN.D7578" / "DN-P 1091" / "DN:HF 974" and this gate + # threw away every one of them — the warp scored 1/21 instead of 4/21 on the + # cam-617 ground-truth set purely because of this line. + # + # The exemption is deliberately narrow: EXACTLY ONE seal-like glyph, strictly + # INTERIOR, and the remainder must CONFORM to the region grammar. Noise + # cannot be laundered by it — the 2026-07-22 junk ("6ou.2x", "[miv4o)") has + # either the wrong count, the wrong position, or no conforming remainder. + if has_symbol_glyph: + repaired = _strip_seal_glyph(text, region) + if repaired is not None: + return { + "kind": "text", + "text": repaired, + "confidence": round(confidence, 4), + "formatValid": True, + } + if alnum_count < min_len or confidence < min_conf or has_symbol_glyph: + return { + "kind": "text", + "text": "", + "confidence": round(confidence, 4), + } + + result = { + "kind": "text", + "text": text, + "confidence": round(confidence, 4), + } + + # Post-hoc format validation + confusable rescoring. 'off' disables it + # entirely (no grammar lookup, no formatValid annotation). + if region != "off": + pattern_src = PLATE_GRAMMARS.get(region) + if pattern_src is not None: + pattern = re.compile(pattern_src) + # Grammar + confusables run on the NORMALISED read (upper, alnum + # only). Reached only past the plausibility gate above, so a repair + # can never turn sub-legible noise into a more credible string: the + # symbol gate already rejected the reads that would qualify, and the + # normalised length equals the alnum count the length gate cleared. + norm_text, norm_scores, norm_timesteps = _normalize_read( + chars, scores, char_timesteps + ) + fixed_text, format_valid = _rescore_for_grammar( + norm_text, norm_scores, norm_timesteps, probs, list(charset), pattern + ) + result["text"] = fixed_text + result["formatValid"] = format_valid + + return result diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/plate_slots.py b/camstack_train/_vendor/pipeline_python/postprocessors/plate_slots.py new file mode 100644 index 0000000000000000000000000000000000000000..70d21f7d622fd463609f4476abde5bb43a714c7d --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/plate_slots.py @@ -0,0 +1,180 @@ +"""Fixed-slot license-plate recognition postprocessor. + +Used for: `cct-s-v2-global` (fast-plate-ocr), the plate-OCR default since +2026-08-20. + +Input: the `plate` head, ``[1, PLATE_SLOTS, VOCAB]`` (or the flattened +``[1, PLATE_SLOTS * VOCAB]``) — an INDEPENDENT softmax per character position, +already normalised inside the graph. +Output: ``{"kind": "text", "text": "DNRO309", "confidence": 0.99, +"formatValid": true}`` — the same StepOutput shape the CTC path produces, so +nothing downstream (the label writer, the D186 per-track vote) changes. + +Why this is not `ctc.py` +------------------------ +The two readers fail in opposite directions and that dictates opposite gates. + +A CTC recognizer fed noise collapses toward the blank: the decode comes out one +or two characters long, which is why "at least `minTextLength` alphanumerics" +was a sound plausibility gate for `vgg_english_g2`. A fixed-slot reader has no +blank. It runs `PLATE_SLOTS` classification heads and every one of them picks a +winner, on a plate, on a bumper, on an OSD weather band. Length carries no +information here — `minTextLength` would admit literally every ROI the detector +ever hands over. + +What DOES separate them is calibration, and only at the weakest position. On the +39-crop cam-617 ground-truth set (21 plates, 9 vehicles, 18 junk ROIs) every +correct read has a minimum per-character probability of 0.80 or better, while no +junk ROI reaches it; the misreads that do get close are wrong in exactly one +slot, and that slot is the one that dips. The MEAN hides this — six confident +glyphs and one guess still average ~0.9 — so the gate reads the MINIMUM. + +Second gate: the region grammar (`plateRegion`, shared with `ctc.py` so there is +one German plate pattern in this codebase, not two). Here a non-conforming read +is DROPPED rather than kept-and-annotated. That differs from the CTC path on +purpose: this alphabet is 36 alphanumerics plus a pad, so it cannot emit the +"obviously not a plate" shapes (`[miv4o)`, `6ou.2`) that made an annotated read +worth storing — a non-conforming read here is a confident misread, which is the +one thing that must never reach a label. + +No confusable rescoring +----------------------- +`ctc.py` repairs a grammar failure by substituting 0↔O, 1↔I, 5↔S… That machinery +exists because `vgg_english_g2` decodes from a 97-glyph English charset where +case and punctuation are live classes. This alphabet has neither, so the only +surviving confusion is 0↔O — and, run over the 39-crop set at the shipped +operating point, the rescorer fires on zero crops. An unfired repair path is a +path nothing verifies, so it is not here. If a future set shows it firing, take +it from `ctc.py`: `_rescore_for_grammar` is already position/probability +indexed, and a slot index is exactly the "aligned timestep" it wants. +""" +import re + +import numpy as np + +from .ctc import ABSENT_PLATE_REGION, PLATE_GRAMMARS + +# The `cct_s_v2_global_plate_config.yaml` contract, published beside the weights. +# Class indices are positions in this string; a drift re-maps every glyph. +PLATE_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_" +PLATE_PAD_CHAR = "_" +PLATE_SLOTS = 10 + +# Minimum probability the WEAKEST emitted character must carry. +# +# 0.75 is not a round number picked for taste — it is the centre of the band +# that separates the two populations on the cam-617 set WITH THE INFERENCE +# PRECISION SPREAD INCLUDED. Taking, per crop, the worst case across ONNX fp32, +# OpenVINO fp16 and CoreML: the highest-scoring junk ROI that also conforms to +# the German grammar reaches 0.728, and the lowest-scoring correct plate that +# still has to be read sits at 0.779. Anything inside (0.728, 0.779) stores +# every one of those plates and none of that junk on every engine. +# +# The band matters because a plate recognizer does not get to choose its +# arithmetic: OpenVINO's CPU default inference precision is fp16, an Intel NPU +# has no other option, and a marginal crop's weakest character moves by as much +# as 0.16 between engines. A floor chosen from one engine's numbers would be a +# different filter on the next node. +# +# The two sides of the band are not symmetric, which is why 0.75 leans up. Too +# LOW stores a wrong plate, and nothing downstream can undo that. Too HIGH +# silences a marginal read — and the step runs up to `maxPerTrack` times per +# track with the reads voted (D186), so the next frame gets another go. +# +# Config-overridable (`minCharProbability`). +DEFAULT_MIN_CHAR_PROBABILITY = 0.75 + +# Name of the plate head in the published graph. The region head (`region`, +# [1,66]) rides along; it is NOT consulted — at this floor every junk ROI it +# would have caught is already below the confidence gate, and a second gate +# nothing removes is a second thing to keep true. +PLATE_OUTPUT_NAME = "plate" + + +def _select_plate_logits(predictions: dict) -> np.ndarray: + """Return the plate head as ``[PLATE_SLOTS, VOCAB]``. + + By NAME first — the graph has two outputs and dict ordering is not a + contract. By SHAPE as the fallback, so an engine whose exporter dropped the + tensor names still decodes the right head rather than the region logits. + """ + vocab = len(PLATE_ALPHABET) + expected = PLATE_SLOTS * vocab + candidate = predictions.get(PLATE_OUTPUT_NAME) + if candidate is None: + for value in predictions.values(): + array = np.asarray(value) + if array.size == expected: + candidate = array + break + if candidate is None: + raise ValueError( + f"no plate head in predictions {sorted(predictions)}: " + f"expected an output of {expected} values" + ) + array = np.asarray(candidate, dtype=np.float32) + if array.size != expected: + raise ValueError( + f"plate head has {array.size} values, expected " + f"{PLATE_SLOTS}x{vocab}={expected}" + ) + return array.reshape(PLATE_SLOTS, vocab) + + +def postprocess_plate_slots( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """Per-slot argmax decode + confidence floor + region grammar.""" + probs = _select_plate_logits(predictions) + alphabet = str(config.get("plateAlphabet") or PLATE_ALPHABET) + pad_char = str(config.get("platePadChar") or PLATE_PAD_CHAR) + min_char_prob = float( + config.get("minCharProbability", DEFAULT_MIN_CHAR_PROBABILITY) + ) + region = str(config.get("plateRegion", ABSENT_PLATE_REGION)) + + winners = np.argmax(probs, axis=-1) + scores = probs[np.arange(len(winners)), winners] + decoded = "".join(alphabet[int(i)] for i in winners) + + # Trailing pad is the model's length signal; strip it and keep the scores of + # the glyphs that survive aligned with them. + text = decoded.rstrip(pad_char) + kept = [float(s) for s in scores[: len(text)]] + confidence = round(float(np.mean(kept)), 4) if kept else 0.0 + + def silent() -> dict: + return {"kind": "text", "text": "", "confidence": confidence} + + if not text: + return silent() + # A pad the model placed BETWEEN glyphs is a refusal at a position it + # believes exists — never a plate, and never something to paper over. + if pad_char in text: + return silent() + if min(kept) < min_char_prob: + return silent() + + pattern_src = PLATE_GRAMMARS.get(region) if region != "off" else None + if pattern_src is None: + # No grammar for this region (or rescoring disabled): the read stands on + # the confidence gate alone, and says so. + return { + "kind": "text", + "text": text, + "confidence": confidence, + "formatValid": False, + } + if re.match(pattern_src, text) is None: + return silent() + return { + "kind": "text", + "text": text, + "confidence": confidence, + "formatValid": True, + } diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/rfdetr.py b/camstack_train/_vendor/pipeline_python/postprocessors/rfdetr.py new file mode 100644 index 0000000000000000000000000000000000000000..7ccfe45cf23fcc9efd9a361ee39032b5c3d067dc --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/rfdetr.py @@ -0,0 +1,161 @@ +"""RF-DETR postprocessor (DETR-style query decode). + +Used by the `rfdetr-package` parcel detector on the `package-detection` step +(per-model `postprocessor: 'rfdetr'` override — the step default is `yolo`). + +Model contract, verified on the shipped export (onnx / openvino / coreml all +agree to |d| <= 0.005 in sigmoid space): + + input "input" [1, 3, 576, 576] float, square RESIZE (no letterbox), + ImageNet mean/std normalised + (catalog: preprocessMode 'resize' + + inputNormalization 'imagenet') + output "dets" [1, 300, 4] cxcywh, NORMALISED to the input image + output "labels" [1, 300, 2] RAW logits; sigmoid -> per-query prob. + +Three things differ from the `yolo` postprocessor and every one of them is a +way to get zero detections while looking correct: + +1. **Column 0 is the no-object slot; the real classes start at 1.** + The upstream rfdetr helper slices `logits[..., :-1]` — it drops the LAST + column. On this checkpoint (`num_classes=1` ⇒ head width 2) that would throw + away the `package` column and keep the background one, so every frame + decodes as "nothing, forever". We slice `[..., 1:]`. Measured: the + background column never exceeded 0.003 on any test frame, parcel or not. + +2. **No letterbox, so `scale`/`pad` are unused.** Boxes map onto the original + crop by multiplying the normalised cxcywh by orig_w / orig_h directly. The + normalised coords are NOT sigmoid-bounded and DO exceed 1.0 (and go below 0) + on edge queries — they are clamped to the frame, and a box that clamps to + nothing is dropped. + +3. **300 independent queries, no objectness.** NMS still runs (near-duplicate + queries do occur) but the model is not anchor-based, so the default IoU is + looser (0.50) than the yolo default on purpose. + +DEGENERATE-BOX GUARD (measured, not theoretical): on a parcel-free crop +dominated by one large dark rectangle (the doormat) the top query becomes a box +spanning ~the whole crop at 0.33. Every real parcel detection was <= 25% of the +crop area. Rejecting boxes covering more than `maxAreaFraction` of the crop +removes that failure mode without touching a single true positive. + +Output matches the yolo postprocessor: +``{"kind": "detections", "detections": [{"class", "score", "bbox":[x1,y1,x2,y2]}]}`` +with pixel bboxes in ORIGINAL frame coordinates. +""" +from __future__ import annotations + +import numpy as np + +# Fraction of the input crop above which a box is treated as the degenerate +# whole-crop query rather than a parcel. Config key: `maxAreaFraction`. +DEFAULT_MAX_AREA_FRACTION = 0.60 +# DETR queries are not anchors, so duplicates overlap less tightly than a yolo +# grid does. Config key: `nmsIouThreshold` (shared with the yolo decoder). +DEFAULT_NMS_IOU = 0.50 + + +def _iou(a: tuple, b: tuple) -> float: + """IoU between two boxes (x1, y1, x2, y2).""" + ix1 = max(a[0], b[0]) + iy1 = max(a[1], b[1]) + ix2 = min(a[2], b[2]) + iy2 = min(a[3], b[3]) + inter = max(0.0, ix2 - ix1) * max(0.0, iy2 - iy1) + if inter == 0: + return 0.0 + union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter + return inter / union if union > 0 else 0.0 + + +def _nms(boxes: list, iou_threshold: float) -> list: + """Greedy non-maximum suppression over `_xyxy`, highest score first.""" + kept: list = [] + for box in sorted(boxes, key=lambda b: b["score"], reverse=True): + if all(_iou(box["_xyxy"], k["_xyxy"]) <= iou_threshold for k in kept): + kept.append(box) + return kept + + +def _resolve_outputs(predictions: dict) -> tuple: + """(dets, labels) by NAME, falling back to shape when the names are stripped. + + The onnx / openvino / coreml exports all keep the `dets` + `labels` names; + the shape fallback covers a re-export that renames them (the boxes tensor is + the one whose last dim is 4). + """ + if "dets" in predictions and "labels" in predictions: + return np.asarray(predictions["dets"]), np.asarray(predictions["labels"]) + arrays = [np.asarray(v) for v in predictions.values()] + dets = next((a for a in arrays if a.shape[-1] == 4), None) + labels = next((a for a in arrays if a.shape[-1] != 4), None) + return dets, labels + + +def postprocess_rfdetr( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple, +) -> dict: + """Decode RF-DETR query output to original-frame detections. + + `scale` / `pad` are accepted for signature parity with the other + postprocessors and deliberately UNUSED: this model is a square resize, not a + letterbox, so there is no padding to invert. + """ + conf_threshold = float(config.get("confidence", 0) or 0) + labels_map = config.get("labels") or ["package"] + nms_iou = float(config.get("nmsIouThreshold", DEFAULT_NMS_IOU) or DEFAULT_NMS_IOU) + max_area = float(config.get("maxAreaFraction", DEFAULT_MAX_AREA_FRACTION) or DEFAULT_MAX_AREA_FRACTION) + + dets, logits = _resolve_outputs(predictions) + if dets is None or logits is None: + return {"kind": "detections", "detections": []} + + dets = dets[0] if dets.ndim == 3 else dets + logits = logits[0] if logits.ndim == 3 else logits + if dets.ndim != 2 or logits.ndim != 2 or logits.shape[1] < 2: + # A single-column `labels` would be the no-object slot alone — nothing + # to decode, and slicing it would raise instead of saying so. + return {"kind": "detections", "detections": []} + + probs = 1.0 / (1.0 + np.exp(-logits.astype(np.float32))) + # Column 0 is the DETR background / no-object slot — NOT the last column. + # See the module docstring: dropping the wrong end silently yields zero + # detections on every frame. + class_probs = probs[:, 1:] + best_class = class_probs.argmax(axis=1) + best_score = class_probs.max(axis=1) + + frame_area = float(orig_w * orig_h) + if frame_area <= 0: + return {"kind": "detections", "detections": []} + + candidates: list = [] + for i in np.nonzero(best_score >= conf_threshold)[0]: + cx, cy, w, h = (float(v) for v in dets[i][:4]) + # Normalised, NOT sigmoid-bounded: edge queries exceed [0,1] and are + # clamped to the frame rather than dropped. + x1 = max(0.0, (cx - w / 2.0) * orig_w) + y1 = max(0.0, (cy - h / 2.0) * orig_h) + x2 = min(float(orig_w), (cx + w / 2.0) * orig_w) + y2 = min(float(orig_h), (cy + h / 2.0) * orig_h) + if x2 <= x1 or y2 <= y1: + continue + if ((x2 - x1) * (y2 - y1)) / frame_area > max_area: + continue # degenerate whole-crop query (the doormat failure mode) + cls = int(best_class[i]) + candidates.append({ + "class": labels_map[cls] if cls < len(labels_map) else str(cls), + "score": round(float(best_score[i]), 4), + "bbox": [round(x1, 1), round(y1, 1), round(x2, 1), round(y2, 1)], + "_xyxy": (x1, y1, x2, y2), + }) + + kept = _nms(candidates, nms_iou) + for box in kept: + box.pop("_xyxy", None) + return {"kind": "detections", "detections": kept} diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/saliency.py b/camstack_train/_vendor/pipeline_python/postprocessors/saliency.py new file mode 100644 index 0000000000000000000000000000000000000000..74e3e21447873c0212523dbd13423cb27dd8d387 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/saliency.py @@ -0,0 +1,90 @@ +"""Saliency / segmentation refiner postprocessor. + +Used for: u2netp. + +Input: raw saliency map tensor +Output: {"kind": "mask", "mask": "", "maskWidth": N, "maskHeight": N} +""" +import base64 +import numpy as np + + +def _largest_component_bbox(binary: np.ndarray) -> tuple[int, int, int, int] | None: + """Bounding box [x, y, w, h] (mask space) of the largest 4-connected + foreground component. Pure-numpy flood fill — no OpenCV dependency. + Returns None when there is no foreground pixel.""" + h, w = binary.shape + fg = binary > 0 + visited = np.zeros((h, w), dtype=bool) + best: tuple[int, int, int, int, int] | None = None # (area, x, y, w, h) + ys, xs = np.nonzero(fg) + for sy, sx in zip(ys.tolist(), xs.tolist()): + if visited[sy, sx]: + continue + stack = [(sy, sx)] + visited[sy, sx] = True + minx = maxx = sx + miny = maxy = sy + area = 0 + while stack: + cy, cx = stack.pop() + area += 1 + if cx < minx: + minx = cx + if cx > maxx: + maxx = cx + if cy < miny: + miny = cy + if cy > maxy: + maxy = cy + for ny, nx in ((cy - 1, cx), (cy + 1, cx), (cy, cx - 1), (cy, cx + 1)): + if 0 <= ny < h and 0 <= nx < w and fg[ny, nx] and not visited[ny, nx]: + visited[ny, nx] = True + stack.append((ny, nx)) + if best is None or area > best[0]: + best = (area, minx, miny, maxx - minx + 1, maxy - miny + 1) + if best is None: + return None + return (best[1], best[2], best[3], best[4]) + + +def postprocess_saliency( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """Sigmoid → threshold → binary mask → base64.""" + mask_threshold = config.get("maskThreshold", 0.5) + + # U2-Net outputs multiple saliency maps; use the first (highest resolution) + raw = np.array(list(predictions.values())[0]).astype(np.float32) + + # Remove batch/channel dims to get [H, W] + while raw.ndim > 2: + raw = raw[0] + + # Sigmoid + saliency = 1.0 / (1.0 + np.exp(-raw)) + + # Threshold to binary uint8 (0 or 255) + binary = (saliency > mask_threshold).astype(np.uint8) * 255 + + h, w = binary.shape + mask_b64 = base64.b64encode(binary.tobytes()).decode("ascii") + + result = { + "kind": "mask", + "mask": mask_b64, + "maskWidth": int(w), + "maskHeight": int(h), + } + + bbox = _largest_component_bbox(binary) + if bbox is not None: + bx, by, bw, bh = bbox + result["maskBbox"] = [int(bx), int(by), int(bw), int(bh)] + + return result diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/scrfd.py b/camstack_train/_vendor/pipeline_python/postprocessors/scrfd.py new file mode 100644 index 0000000000000000000000000000000000000000..7e3f384b1e434aa17e2d71daa7980895db3dfff6 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/scrfd.py @@ -0,0 +1,235 @@ +"""SCRFD face detection postprocessor. + +Multi-stride anchor-based face detector (strides 8, 16, 32). +Originally ported from packages/addon-vision/src/shared/postprocess/scrfd.ts, +whose Node.js raw-tensor onnxruntime path was retired in favour of doing all +postprocessing here, in the Python inference pool — that TS file, and the +addon-vision package it lived in, no longer exist (see the header comment in +packages/addon-pipeline/src/detection-pipeline/postprocess/dispatch.ts). This +module is now the sole SCRFD decoder in the repo. + +Output: {"kind": "detections", "detections": [{"class": "face", "score", "bbox": [x1,y1,x2,y2], "landmarks?"}]} +""" + +from __future__ import annotations +import numpy as np + + +STRIDES = [8, 16, 32] +NUM_ANCHORS_PER_STRIDE = 2 + + +def _generate_anchors(stride: int, input_size: int) -> list[tuple[float, float]]: + """Generate anchor centers for a given stride. + + Anchor centers are `(x * stride, y * stride)` — NO half-cell `+0.5` offset. + This matches upstream InsightFace's reference decoder exactly + (`insightface/model_zoo/scrfd.py`: `anchor_centers = np.stack([anchor_centers_x, + anchor_centers_y], axis=-1).astype(np.float32)`, built from a raw + `np.mgrid[:h, :w]` and then `anchor_centers * stride` — no `+0.5` anywhere + in that path). The previous `(x + 0.5) * stride` formula shifted every box + and landmark down-and-right by half a stride (4/8/16 px at 640), which is a + large fraction of a face after the letterbox upscale from a person crop. + Fixing this moved NME (COCO eyes+nose vs GT) 0.352 → 0.115 and AuraFace + self-consistency cosine 0.79 → 0.91 (2026-09-26 model-replacement spike, + docs/superpowers/specs/2026-09-26-replacement-models-spike.md §2.4-1). + """ + feat_size = int(np.ceil(input_size / stride)) + anchors = [] + for y in range(feat_size): + for x in range(feat_size): + for _ in range(NUM_ANCHORS_PER_STRIDE): + anchors.append((x * stride, y * stride)) + return anchors + + +def _iou(a: tuple, b: tuple) -> float: + ax1, ay1, ax2, ay2 = a + bx1, by1, bx2, by2 = b + ix1, iy1 = max(ax1, bx1), max(ay1, by1) + ix2, iy2 = min(ax2, bx2), min(ay2, by2) + inter = max(0, ix2 - ix1) * max(0, iy2 - iy1) + if inter == 0: + return 0.0 + union = (ax2 - ax1) * (ay2 - ay1) + (bx2 - bx1) * (by2 - by1) - inter + return inter / union if union > 0 else 0.0 + + +def _nms(boxes: list[dict], iou_threshold: float = 0.45) -> list[dict]: + if not boxes: + return [] + sorted_boxes = sorted(boxes, key=lambda b: b["score"], reverse=True) + kept = [] + suppressed = set() + for i, box in enumerate(sorted_boxes): + if i in suppressed: + continue + kept.append(box) + for j in range(i + 1, len(sorted_boxes)): + if j in suppressed: + continue + if _iou(box["_xyxy"], sorted_boxes[j]["_xyxy"]) > iou_threshold: + suppressed.add(j) + return kept + + +def _match_outputs_to_strides( + raw_outputs: dict[str, np.ndarray], + strides: list[int], + input_size: int, +) -> dict[int, dict]: + """Match model output tensors to strides. + + Handles two naming conventions: + 1. Named keys: 'score_8', 'bbox_8', 'kps_8' (ONNX/OpenVINO) + 2. Generic keys: 'var_732' etc. — matched by shape (CoreML) + + For CoreML, each stride has 3 tensors with num_anchors matching the stride: + stride 8 → ceil(input/8)^2 * 2 anchors → shapes (1, N, 1), (1, N, 4), (1, N, 10) + stride 16 → ceil(input/16)^2 * 2 + stride 32 → ceil(input/32)^2 * 2 + """ + result: dict[int, dict] = {} + + # Try named keys first + flat = {k: v.flatten() for k, v in raw_outputs.items()} + for stride in strides: + score_key = bbox_key = kps_key = None + for k in flat: + if f"score_{stride}" in k or f"_{stride}_score" in k: + score_key = k + elif f"bbox_{stride}" in k or f"_{stride}_bbox" in k: + bbox_key = k + elif f"kps_{stride}" in k or f"_{stride}_kps" in k: + kps_key = k + if score_key and bbox_key: + result[stride] = { + "scores": flat[score_key], + "bboxes": flat[bbox_key], + "kps": flat.get(kps_key), + } + + if len(result) == len(strides): + return result + + # Fallback: match by shape (CoreML generic names) + # Group outputs by last dimension: 1=scores, 4=bboxes, 10=landmarks + scores_by_n: dict[int, np.ndarray] = {} + bboxes_by_n: dict[int, np.ndarray] = {} + kps_by_n: dict[int, np.ndarray] = {} + + for _k, arr in raw_outputs.items(): + if arr.ndim == 3: + arr = arr[0] # remove batch dim → (N, C) + elif arr.ndim == 1: + continue + n, c = arr.shape + if c == 1: + scores_by_n[n] = arr.flatten() + elif c == 4: + bboxes_by_n[n] = arr.flatten() + elif c == 10: + kps_by_n[n] = arr.flatten() + + result = {} + for stride in strides: + feat = int(np.ceil(input_size / stride)) + expected_n = feat * feat * NUM_ANCHORS_PER_STRIDE + if expected_n in scores_by_n and expected_n in bboxes_by_n: + result[stride] = { + "scores": scores_by_n[expected_n], + "bboxes": bboxes_by_n[expected_n], + "kps": kps_by_n.get(expected_n), + } + + return result + + +def postprocess_scrfd( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """Postprocess SCRFD output to structured face detections.""" + conf_threshold = float(config.get("confidence", 0.5) or 0.5) + input_size = config.get("inputSize", 640) + strides = config.get("strides", STRIDES) + # NMS IoU from the step config (operator slider); the 0.45 literal stays as + # the fallback so an unset value is byte-identical to today (#4). + nms_iou = float(config.get("nmsIouThreshold", 0.45)) + + # Letterbox transform: scale and pad are passed from inference_pool + # scale = input_size / max(orig_w, orig_h) + # pad = ((input_size - orig_w*scale)//2, (input_size - orig_h*scale)//2) + + # Convert all outputs to numpy arrays (keep ndim) + raw_outputs = {k: np.array(v) for k, v in predictions.items()} + + # Match outputs to strides — by named key or by shape + stride_data = _match_outputs_to_strides(raw_outputs, strides, input_size) + + candidates = [] + + for stride in strides: + data = stride_data.get(stride) + if data is None: + continue + + scores = data["scores"] + bboxes = data["bboxes"] + kps = data.get("kps") + anchors = _generate_anchors(stride, input_size) + + n = len(anchors) + for i in range(n): + if i >= len(scores): + break + score = float(scores[i]) + if score < conf_threshold: + continue + + cx, cy = anchors[i] + + # Bbox: offsets from anchor center, scaled by stride + x1 = cx - float(bboxes[i * 4]) * stride + y1 = cy - float(bboxes[i * 4 + 1]) * stride + x2 = cx + float(bboxes[i * 4 + 2]) * stride + y2 = cy + float(bboxes[i * 4 + 3]) * stride + + # Transform from letterbox coords to original image + ox1 = max(0, min(orig_w, (x1 - pad[0]) / scale)) + oy1 = max(0, min(orig_h, (y1 - pad[1]) / scale)) + ox2 = max(0, min(orig_w, (x2 - pad[0]) / scale)) + oy2 = max(0, min(orig_h, (y2 - pad[1]) / scale)) + + if (ox2 - ox1) < 1 or (oy2 - oy1) < 1: + continue + + det = { + "class": "face", + "score": round(score, 4), + "bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)], + "_xyxy": (ox1, oy1, ox2, oy2), + } + + # Landmarks (5 points × 2 coords) + if kps is not None and (i * 10 + 9) < len(kps): + landmarks = [] + for p in range(5): + raw_lx = cx + float(kps[i * 10 + p * 2]) * stride + raw_ly = cy + float(kps[i * 10 + p * 2 + 1]) * stride + lx = (raw_lx - pad[0]) / scale + ly = (raw_ly - pad[1]) / scale + landmarks.append({"x": round(lx, 1), "y": round(ly, 1)}) + det["landmarks"] = landmarks + + candidates.append(det) + + kept = _nms(candidates, nms_iou) + for d in kept: + del d["_xyxy"] + + return {"kind": "detections", "detections": kept} diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/softmax.py b/camstack_train/_vendor/pipeline_python/postprocessors/softmax.py new file mode 100644 index 0000000000000000000000000000000000000000..590f76319091b1d5047becc959c913cfa687c71f --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/softmax.py @@ -0,0 +1,56 @@ +"""Softmax classifier postprocessor. + +Used for: animal-classifier, bird-classifier, vehicle-type-v1, and any custom +softmax classifier. + +Input: raw logits tensor +Output: {"kind": "classifications", "classifications": [{"class", "score"}]} +""" +import numpy as np + + +def postprocess_softmax( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """Softmax + argmax + label lookup. + + When ``config['outputProbabilities']`` is set the model already applies + softmax IN-GRAPH (e.g. the Google AIY Birds classifier) — its raw output is + a probability distribution, so we consume it directly. Re-softmaxing an + already-normalised probability vector is NOT a no-op: it collapses the + distribution toward uniform (the top-1 score craters far below its true + value), which silently defeats every downstream confidence gate. + """ + labels = config.get("labels", []) + + # Get first output tensor + raw = np.array(list(predictions.values())[0]).flatten().astype(np.float32) + + if config.get("outputProbabilities"): + # Model output is already a softmax distribution — do NOT re-softmax. + probs = raw + else: + # Stable softmax over raw logits. + shifted = raw - np.max(raw) + exps = np.exp(shifted) + probs = exps / np.sum(exps) + + # Return top-K classifications — normalization (top-1 + alternates) happens in TypeScript + top_k = min(5, len(probs)) + top_indices = np.argsort(probs)[::-1][:top_k] + + classifications = [] + for idx in top_indices: + idx = int(idx) + score = float(probs[idx]) + if score < 0.01: + break + label = labels[idx] if idx < len(labels) else str(idx) + classifications.append({"class": label, "score": round(score, 4)}) + + return {"kind": "classifications", "classifications": classifications} diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/ssd.py b/camstack_train/_vendor/pipeline_python/postprocessors/ssd.py new file mode 100644 index 0000000000000000000000000000000000000000..c96ab1d1842ed16860a004ea6ee05fccca4dcac8 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/ssd.py @@ -0,0 +1,184 @@ +"""SSD / EfficientDet-Lite postprocessor (TFLite_Detection_PostProcess op). + +Used by the EdgeTPU (Coral USB Edge TPU) runtime with the Coral model-zoo +SSD MobileNet V2 and EfficientDet-Lite detectors. Those models embed the +``TFLite_Detection_PostProcess`` op, so the 4 output tensors are ALREADY +decoded (boxes + per-box class + score, post-NMS): + + output 0 boxes [1, N, 4] (ymin, xmin, ymax, xmax) normalised 0..1 + output 1 classes [1, N] float class indices (COCO 90-class map) + output 2 scores [1, N] float 0..1 + output 3 count [1] number of valid detections + +The inference_pool ``edgetpu`` predict closure returns them keyed by their +positional output index ("0".."3"). Output matches the yolo postprocessor: +``{"kind": "detections", "detections": [{"class", "score", "bbox":[x1,y1,x2,y2]}]}`` +with pixel bboxes in ORIGINAL frame coordinates. + +The same layout, verified per model on the upstream graph (2026-09-26): + + ssd-mobilenet-v2-coco-edgetpu 300x300 uint8, N=20 + efficientdet-lite0-edgetpu 320x320 uint8, N=25 + ssdlite-mobiledet-coco-edgetpu 320x320 uint8, N=100, 91 score columns + (background + COCO-90), so the emitted + class index is already 0-based: 0 = person + +The decode is plain Python over nested lists. Each tensor is turned into a list +once (``ndarray.tolist()``), so this module needs no numpy: CI's self-hosted +runner has none, and the MobileDet fixture test (``test_ssd.py``) must run +there, not be skipped there. Measured: ~85 µs per frame against ~14 µs for +the old numpy decode (about 6×), under 1% of a core at 100 fps and small next +to a ~9 ms Coral invoke. +""" +from __future__ import annotations + +from typing import Any + +# Coral COCO 90-class label map (github.com/google-coral/test_data coco_labels.txt). +# The "n/a" placeholders keep the raw model class indices aligned and are skipped. +COCO_90 = [ + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", + "boat", "traffic light", "fire hydrant", "n/a", "stop sign", "parking meter", + "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", + "zebra", "giraffe", "n/a", "backpack", "umbrella", "n/a", "n/a", "handbag", + "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", + "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", + "bottle", "n/a", "wine glass", "cup", "fork", "knife", "spoon", "bowl", + "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", + "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "n/a", + "dining table", "n/a", "n/a", "toilet", "n/a", "tv", "laptop", "mouse", + "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", + "refrigerator", "n/a", "book", "clock", "vase", "scissors", "teddy bear", + "hair drier", "toothbrush", +] + + +def _as_list(value: Any) -> Any: + """A tensor as nested Python lists (a scalar stays a scalar).""" + return value.tolist() if hasattr(value, "tolist") else value + + +def _shape(value: Any) -> tuple: + """numpy-style shape of a nested list; ``()`` for a scalar.""" + dims: list[int] = [] + while isinstance(value, (list, tuple)): + dims.append(len(value)) + if not value: + break + value = value[0] + return tuple(dims) + + +def _flatten(value: Any) -> list: + """Every scalar of a nested list, in row-major order (``reshape(-1)``).""" + if not isinstance(value, (list, tuple)): + return [value] + flat: list = [] + for item in value: + flat.extend(_flatten(item)) + return flat + + +def _rows_of_4(value: Any) -> list: + """``reshape(-1, 4)`` of a nested list; refuses a size numpy would refuse.""" + flat = _flatten(value) + if len(flat) % 4 != 0: + raise ValueError(f"cannot reshape {len(flat)} box values into rows of 4") + return [flat[i:i + 4] for i in range(0, len(flat), 4)] + + +def _split_outputs(predictions: dict) -> tuple: + """Resolve (boxes, classes, scores, count) from the raw prediction dict. + + Primary path: positional keys "0".."3" (the edgetpu predict closure). + Fallback: shape-based identification for name-keyed / reordered exports. + """ + if all(k in predictions for k in ("0", "1", "2", "3")): + return ( + _as_list(predictions["0"]), + _as_list(predictions["1"]), + _as_list(predictions["2"]), + _as_list(predictions["3"]), + ) + vals = [_as_list(v) for v in predictions.values()] + boxes = count = None + two_d: list = [] + for v in vals: + shape = _shape(v) + ndim = len(shape) + if ndim >= 2 and shape[-1] == 4: + boxes = v + elif ndim == 1 or (ndim == 2 and shape[-1] == 1): + count = v + elif ndim == 2: + two_d.append(v) + # TFLite_Detection_PostProcess emits classes BEFORE scores. + classes = two_d[0] if len(two_d) > 0 else None + scores = two_d[1] if len(two_d) > 1 else None + return boxes, classes, scores, count + + +def label_for_class(cls: int) -> str: + """COCO-90 label of a raw class index; ``"n/a"`` for a gap, ``str`` if outside.""" + return COCO_90[cls] if 0 <= cls < len(COCO_90) else str(cls) + + +def postprocess_ssd( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple, +) -> dict: + """Decode SSD/EfficientDet postprocess outputs to original-frame detections.""" + conf_threshold = float(config.get("confidence", 0.4) or 0.4) + input_size = int(config.get("inputSize", 300) or 300) + + boxes, classes, scores, count = _split_outputs(predictions) + if boxes is None or scores is None: + return {"kind": "detections", "detections": []} + + boxes = _rows_of_4(boxes) + scores = _flatten(scores) + classes = _flatten(classes) if classes is not None else [0.0] * len(scores) + n = len(scores) + count_flat = _flatten(count) if count is not None else [] + if count_flat: + n = min(n, int(count_flat[0])) + n = min(n, len(boxes), len(classes)) + + pad_x, pad_y = pad + is_letterbox = not (abs(scale - 1.0) < 1e-6 and pad_x == 0 and pad_y == 0) + + detections = [] + for i in range(n): + score = float(scores[i]) + if score < conf_threshold: + continue + ymin, xmin, ymax, xmax = (float(v) for v in boxes[i]) + if is_letterbox: + # Invert letterbox: normalised -> input px -> remove pad -> unscale. + x1 = (xmin * input_size - pad_x) / scale + y1 = (ymin * input_size - pad_y) / scale + x2 = (xmax * input_size - pad_x) / scale + y2 = (ymax * input_size - pad_y) / scale + else: + # Plain resize: normalised coords map linearly to the original frame. + x1, y1, x2, y2 = xmin * orig_w, ymin * orig_h, xmax * orig_w, ymax * orig_h + x1 = max(0.0, min(x1, orig_w)) + x2 = max(0.0, min(x2, orig_w)) + y1 = max(0.0, min(y1, orig_h)) + y2 = max(0.0, min(y2, orig_h)) + if x2 <= x1 or y2 <= y1: + continue + label = label_for_class(int(classes[i])) + if label == "n/a": + continue + detections.append({ + "class": label, + "score": round(score, 4), + "bbox": [round(x1, 1), round(y1, 1), round(x2, 1), round(y2, 1)], + }) + + return {"kind": "detections", "detections": detections} diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/yamnet.py b/camstack_train/_vendor/pipeline_python/postprocessors/yamnet.py new file mode 100644 index 0000000000000000000000000000000000000000..5a404ca4f128a3649ac66be4ad09ba447df12550 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/yamnet.py @@ -0,0 +1,41 @@ +"""YAMNet audio classification postprocessor. + +Input: [numFrames, 521] score matrix +Output: {"kind": "classifications", "classifications": [{"class", "score"}]} +""" +import numpy as np + + +def postprocess_yamnet( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """Average scores across frames, return top classes above threshold.""" + labels = config.get("labels", []) + min_score = config.get("confidence", 0.1) + + # Get output_0 (scores) — shape [numFrames, 521] + raw = np.array(list(predictions.values())[0]).astype(np.float32) + if raw.ndim == 1: + raw = raw.reshape(1, -1) + + num_frames, num_classes = raw.shape + + # Average across frames + avg_scores = np.mean(raw, axis=0) + + # Collect classes above threshold, sorted by score + results = [] + for c in range(num_classes): + score = float(avg_scores[c]) + if score >= min_score: + label = labels[c] if c < len(labels) else str(c) + results.append({"class": label, "score": round(score, 4)}) + + results.sort(key=lambda x: x["score"], reverse=True) + + return {"kind": "classifications", "classifications": results} diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/yolo.py b/camstack_train/_vendor/pipeline_python/postprocessors/yolo.py new file mode 100644 index 0000000000000000000000000000000000000000..0653815bcc736eabdc92a0b11d7a43d5dfc76edc --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/yolo.py @@ -0,0 +1,364 @@ +"""YOLO v8/v9/v11 postprocessor. + +Input: raw predictions dict from CoreML/OpenVINO/ONNX +Output: {"kind": "detections", "detections": [{"class", "score", "bbox": [x1,y1,x2,y2]}]} + +Handles both standard and built-in NMS outputs. +""" +import numpy as np + +COCO_80 = [ + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", + "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", + "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", + "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", + "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", + "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", + "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", + "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", + "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", + "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", + "toothbrush", +] + + +def _iou(a: tuple, b: tuple) -> float: + """IoU between two boxes (x1, y1, x2, y2).""" + ax1, ay1, ax2, ay2 = a + bx1, by1, bx2, by2 = b + ix1 = max(ax1, bx1) + iy1 = max(ay1, by1) + ix2 = min(ax2, bx2) + iy2 = min(ay2, by2) + inter = max(0, ix2 - ix1) * max(0, iy2 - iy1) + if inter == 0: + return 0.0 + area_a = (ax2 - ax1) * (ay2 - ay1) + area_b = (bx2 - bx1) * (by2 - by1) + union = area_a + area_b - inter + return inter / union if union > 0 else 0.0 + + +def _nms(boxes: list[dict], iou_threshold: float = 0.45) -> list[dict]: + """Non-maximum suppression. Input: list of {bbox, score, class}.""" + if not boxes: + return [] + sorted_boxes = sorted(boxes, key=lambda b: b["score"], reverse=True) + kept = [] + suppressed = set() + for i, box in enumerate(sorted_boxes): + if i in suppressed: + continue + kept.append(box) + for j in range(i + 1, len(sorted_boxes)): + if j in suppressed: + continue + if _iou(box["_xyxy"], sorted_boxes[j]["_xyxy"]) > iou_threshold: + suppressed.add(j) + return kept + + +# The raw YOLOX head of the `yolox-tiny-plate` Edge TPU build (D710): the +# graph stops before YOLOX's grid decode (an `Exp` and a coordinate Concat would +# share one int8 scale between pixel coordinates and probabilities) and emits, +# per stride s, `reg_s` = (dx, dy, log w, log h) in stride units and +# `score_s` = obj*cls, NHWC as the TFLite file stores them. Decoded here to the +# YOLOv8 layout every other build of the model already emits, so the rest of +# this postprocessor is shared. Only the Edge TPU file carries these names. +YOLOX_HEAD_STRIDES: tuple[int, ...] = (8, 16, 32) + + +def yolox_head_keys() -> list[str]: + return [f"{kind}_{s}" for s in YOLOX_HEAD_STRIDES for kind in ("reg", "score")] + + +def yolox_head_to_v8(predictions: dict, input_size: int) -> "np.ndarray | None": + """``[1, 5, N]`` (cx, cy, w, h, score) from a raw YOLOX head, or ``None`` + when ``predictions`` is not one. RAISES on a head sized for another input: + decoding it would put every prior in the wrong cell.""" + if not all(k in predictions for k in yolox_head_keys()): + return None + columns = [] + for s in YOLOX_HEAD_STRIDES: + n = input_size // s + reg = np.asarray(predictions[f"reg_{s}"], dtype=np.float32).reshape(-1) + score = np.asarray(predictions[f"score_{s}"], dtype=np.float32).reshape(-1) + if reg.size != n * n * 4 or score.size != n * n: + raise ValueError( + f"yolox head: stride {s} has {reg.size} reg / {score.size} score values, " + f"expected {n * n * 4} / {n * n} for inputSize {input_size}" + ) + reg = reg.reshape(n, n, 4) + rows, cols = np.meshgrid(np.arange(n, dtype=np.float32), np.arange(n, dtype=np.float32), indexing="ij") + cx = (reg[..., 0] + cols) * s + cy = (reg[..., 1] + rows) * s + w = np.exp(reg[..., 2]) * s + h = np.exp(reg[..., 3]) * s + columns.append(np.stack([cx, cy, w, h, score.reshape(n, n)], axis=0).reshape(5, -1)) + return np.concatenate(columns, axis=1)[np.newaxis] + + +def _postprocess_coreml_nms( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + input_size: int, +) -> dict: + """Handle CoreML models with built-in NMS. + + CoreML NMS output has two tensors: + - 'confidence': (N, numClasses) — class scores post-NMS + - 'coordinates': (N, 4) — [cx, cy, w, h] normalized to input image size + """ + conf_threshold = float(config.get("confidence", 0)) + labels = config.get("labels", COCO_80) + + confidence = np.array(predictions["confidence"]) + coordinates = np.array(predictions["coordinates"]) + + if confidence.ndim == 1: + confidence = confidence.reshape(1, -1) + if coordinates.ndim == 1: + coordinates = coordinates.reshape(1, -1) + + detections = [] + for i in range(confidence.shape[0]): + best_class = int(np.argmax(confidence[i])) + best_score = float(confidence[i, best_class]) + + if best_score < conf_threshold: + continue + + # CoreML coordinates: [cx, cy, w, h] normalized to input_size + cx, cy, bw, bh = coordinates[i] + # Scale to input pixel coords + cx_px = cx * input_size + cy_px = cy * input_size + bw_px = bw * input_size + bh_px = bh * input_size + + x1 = cx_px - bw_px / 2 + y1 = cy_px - bh_px / 2 + x2 = cx_px + bw_px / 2 + y2 = cy_px + bh_px / 2 + + # Map back to original image coords (undo letterbox) + # CoreML imageType with letterbox: the model sees the padded image + # so we need to figure out the letterbox transform + scale = input_size / max(orig_w, orig_h) + new_w = int(orig_w * scale) + new_h = int(orig_h * scale) + pad_x = (input_size - new_w) / 2 + pad_y = (input_size - new_h) / 2 + + ox1 = max(0, min(orig_w, (x1 - pad_x) / scale)) + oy1 = max(0, min(orig_h, (y1 - pad_y) / scale)) + ox2 = max(0, min(orig_w, (x2 - pad_x) / scale)) + oy2 = max(0, min(orig_h, (y2 - pad_y) / scale)) + + label = labels[best_class] if best_class < len(labels) else str(best_class) + detections.append({ + "class": label, + "score": round(best_score, 4), + "bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)], + }) + + return {"kind": "detections", "detections": detections} + + +def _postprocess_yolo11_nms( + output: np.ndarray, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """Handle YOLO11/v26 post-NMS output: [N, 6] — (x1, y1, x2, y2, score, class_id). + + Coordinates are in input pixel space (0..inputSize). Already post-NMS. + """ + rows = decode_yolo11_nms_rows(output, config, orig_w, orig_h, scale, pad) + return {"kind": "detections", "detections": [d for _, d in rows]} + + +def decode_yolo11_nms_rows( + output: np.ndarray, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> list[tuple[int, dict]]: + """The post-NMS box decode, each kept detection paired with its ROW in + ``output``. Only the first 6 columns are read, so a seg head's + ``[N, 6 + 32]`` rows decode exactly like the plain detector's ``[N, 6]`` + (D737) and the row index finds the detection's mask coefficients.""" + conf_threshold = float(config.get("confidence", 0)) + labels = config.get("labels", COCO_80) + + detections: list[tuple[int, dict]] = [] + for i in range(output.shape[0]): + x1, y1, x2, y2, score, class_id = output[i][:6] + score = float(score) + if score < conf_threshold: + continue + class_id = int(class_id) + if class_id < 0: + continue + + # Transform from letterbox pixel coords to original image + ox1 = max(0, min(orig_w, (float(x1) - pad[0]) / scale)) + oy1 = max(0, min(orig_h, (float(y1) - pad[1]) / scale)) + ox2 = max(0, min(orig_w, (float(x2) - pad[0]) / scale)) + oy2 = max(0, min(orig_h, (float(y2) - pad[1]) / scale)) + + label = labels[class_id] if class_id < len(labels) else str(class_id) + detections.append((i, { + "class": label, + "score": round(score, 4), + "bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)], + })) + + return detections + + +def postprocess_yolo( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """Postprocess YOLO output to structured detections. + + Handles two formats: + 1. Raw tensor (ONNX/OpenVINO): [4+numClasses, numBoxes] — needs NMS + 2. CoreML built-in NMS: 'confidence' + 'coordinates' tensors — already post-NMS + """ + labels = config.get("labels", COCO_80) + num_classes = config.get("numClasses", len(labels)) + + yolox = yolox_head_to_v8(predictions, int(config.get("inputSize", 640))) + if yolox is not None: + predictions = {"output0": yolox} + + # Detect CoreML NMS output format + if "confidence" in predictions and "coordinates" in predictions: + input_size = config.get("inputSize", 640) + return _postprocess_coreml_nms(predictions, config, orig_w, orig_h, input_size) + + # Detect YOLO11/v26 post-NMS format: [batch, N, 6] — (x1, y1, x2, y2, score, class_id) + first_arr = np.array(list(predictions.values())[0]) + if first_arr.ndim == 3: + first_arr = first_arr[0] # remove batch dim + if first_arr.ndim == 2 and first_arr.shape[1] == 6: + return _postprocess_yolo11_nms(first_arr, config, orig_w, orig_h, scale, pad) + + # Raw tensor path (ONNX / OpenVINO / CoreML without NMS) + output = None + for key, val in predictions.items(): + arr = np.array(val) + if arr.ndim >= 2 and any(d in (4 + num_classes, 84, 116, 144) for d in arr.shape): + output = arr + break + if output is None: + output = np.array(list(predictions.values())[0]) + + # Handle various shapes + if output.ndim == 3: + output = output[0] # remove batch dim + + # YOLO output: [4+numClasses, numBoxes] or [numBoxes, 4+numClasses] + if output.shape[0] == 4 + num_classes: + pass # already [C, N] + elif output.shape[1] == 4 + num_classes: + output = output.T # transpose to [C, N] + else: + # Try to infer + if output.shape[0] < output.shape[1]: + pass # likely [C, N] + else: + output = output.T + + rows = decode_raw_yolo_columns(output, config, orig_w, orig_h, scale, pad) + return {"kind": "detections", "detections": [d for _, d in rows]} + + +def decode_raw_yolo_columns( + output: np.ndarray, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> list[tuple[int, dict]]: + """The raw-head box decode (``[4 + numClasses (+ extra), numBoxes]``, one + COLUMN per candidate): sigmoid when the scores are logits, confidence + floor, NMS. Each kept detection is paired with its column in ``output``. + Rows past ``4 + numClasses`` are never read, so a seg head's mask + coefficients ride along untouched and decode exactly like the plain + detector (D737). May apply the sigmoid IN PLACE on the class rows.""" + conf_threshold = float(config.get("confidence", 0)) + labels = config.get("labels", COCO_80) + num_classes = config.get("numClasses", len(labels)) + # NMS IoU from the step config (operator slider); the 0.45 literal stays as + # the fallback so an unset value is byte-identical to today (#4). + nms_iou = float(config.get("nmsIouThreshold", 0.45)) + + num_boxes = output.shape[1] + + # Detect if scores are raw logits (YOLO11/v26) vs sigmoid-activated (YOLOv8/v9). + # Sigmoid-activated scores are in [0, 1]. If any class score exceeds 1.0, + # the model outputs raw logits and we need to apply sigmoid. + sample_scores = output[4:4 + num_classes, :min(100, num_boxes)] + needs_sigmoid = float(np.max(sample_scores)) > 1.0 or float(np.min(sample_scores)) < 0.0 + if needs_sigmoid: + # Apply sigmoid to all class scores in-place (rows 4..4+numClasses) + output[4:4 + num_classes, :] = 1.0 / (1.0 + np.exp(-np.clip(output[4:4 + num_classes, :], -50, 50))) + + # Vectorised candidate extraction — replaces the per-box Python loop + # that took ~20ms on 8400 boxes. Pure numpy: ~0.5ms. + class_scores = output[4:4 + num_classes, :] # (C, N) + best_classes = np.argmax(class_scores, axis=0) # (N,) + best_scores = class_scores[best_classes, np.arange(num_boxes)] # (N,) + mask = best_scores >= conf_threshold + if not np.any(mask): + return [] + + # Gather only above-threshold boxes + idxs = np.nonzero(mask)[0] + cx = output[0, idxs] + cy = output[1, idxs] + bw = output[2, idxs] + bh = output[3, idxs] + sc = best_scores[idxs] + cl = best_classes[idxs] + + # Centre → corner, letterbox → original coords + x1 = np.clip((cx - bw / 2 - pad[0]) / scale, 0, orig_w) + y1 = np.clip((cy - bh / 2 - pad[1]) / scale, 0, orig_h) + x2 = np.clip((cx + bw / 2 - pad[0]) / scale, 0, orig_w) + y2 = np.clip((cy + bh / 2 - pad[1]) / scale, 0, orig_h) + + candidates = [] + for k in range(len(idxs)): + ci = int(cl[k]) + candidates.append({ + "class": labels[ci] if ci < len(labels) else str(ci), + "score": round(float(sc[k]), 4), + "bbox": [round(float(x1[k]), 1), round(float(y1[k]), 1), round(float(x2[k]), 1), round(float(y2[k]), 1)], + "_xyxy": (float(x1[k]), float(y1[k]), float(x2[k]), float(y2[k])), + "_col": int(idxs[k]), + }) + + kept = _nms(candidates, nms_iou) + # Remove the internal fields; the column travels beside the detection. + out: list[tuple[int, dict]] = [] + for d in kept: + del d["_xyxy"] + out.append((d.pop("_col"), d)) + return out diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/yolo_seg.py b/camstack_train/_vendor/pipeline_python/postprocessors/yolo_seg.py new file mode 100644 index 0000000000000000000000000000000000000000..919ff138ae0ce861e2d8eb56fe60a150810cb2c6 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/yolo_seg.py @@ -0,0 +1,173 @@ +"""YOLO-seg postprocessor (YOLO26-seg instance segmentation). + +Input: raw predictions dict with two outputs: + - detections: post-NMS ``[1, 300, 6 + 32]`` rows + ``[x1, y1, x2, y2, conf, class_id, coeff_0..coeff_31]``, or a raw head + ``[1, 4 + numClasses + 32, numBoxes]`` (one column per candidate) + - prototypes: ``[1, 32, P, P]`` (P = 160 at a 640 input) + +Output: ``{"kind": "detections", "detections": [{..., "mask": "", +"maskWidth": N, "maskHeight": N}]}`` — the mask is the detection's box region +of the prototype grid, thresholded to 0/255 bytes, row-major, base64. + +The BOXES are decoded by the plain detector's own functions +(``yolo.decode_yolo11_nms_rows`` / ``yolo.decode_raw_yolo_columns``): same +confidence floor, same sigmoid detection, same NMS (and none on a post-NMS +head, exactly like ``yolo``), same letterbox inversion and rounding. A seg +model chosen as the camera ROOT therefore produces the detections a plain +detector would from the same tensors, plus a mask on each (D737). Masks are +computed only for the detections that survive, and only over their box. +""" +import base64 + +import numpy as np + +from .yolo import COCO_80, decode_raw_yolo_columns, decode_yolo11_nms_rows + +NUM_MASK_COEFFS = 32 +PROTO_SIZE = 160 + + +def _sigmoid(x: np.ndarray) -> np.ndarray: + """Numerically stable sigmoid.""" + return np.where(x >= 0, 1 / (1 + np.exp(-x)), np.exp(x) / (1 + np.exp(x))) + + +def _as_prototypes(arr: np.ndarray) -> np.ndarray | None: + """``[32, P, P]`` when ``arr`` is a prototype tensor, else ``None``.""" + squeezed = arr + while squeezed.ndim > 3 and squeezed.shape[0] == 1: + squeezed = squeezed[0] + if squeezed.ndim == 3 and squeezed.shape[0] == NUM_MASK_COEFFS and squeezed.shape[1] == squeezed.shape[2]: + return squeezed + if arr.size == NUM_MASK_COEFFS * PROTO_SIZE * PROTO_SIZE: + return arr.reshape(NUM_MASK_COEFFS, PROTO_SIZE, PROTO_SIZE) + return None + + +def _find_tensors(predictions: dict, num_classes: int) -> tuple[np.ndarray, np.ndarray]: + """Locate the detection tensor and prototype tensor from model outputs. + + Returns (detections, prototypes) as numpy arrays. + detections: [num_dets, channels] row-major + prototypes: [32, P, P] + """ + det_tensor = None + proto_tensor = None + + for _key, val in predictions.items(): + arr = np.array(val, dtype=np.float32) + + protos = _as_prototypes(arr) + if protos is not None and proto_tensor is None: + proto_tensor = protos + continue + + # Skip tiny tensors + if arr.size < 10: + continue + + # Detection tensor: the other large tensor + if det_tensor is None: + det_tensor = arr + + if det_tensor is None: + raise ValueError("YOLO-seg postprocessor: could not find detection tensor") + if proto_tensor is None: + raise ValueError("YOLO-seg postprocessor: could not find prototype tensor") + + # Normalize detection tensor to [num_dets, channels] + while det_tensor.ndim > 2: + det_tensor = det_tensor[0] # strip batch dim + + # NMS format: [300, 38] — rows are detections + # Raw format: [4+nc+32, N] — cols are detections, needs transpose + nms_cols = 6 + NUM_MASK_COEFFS # 38 + raw_cols = 4 + num_classes + NUM_MASK_COEFFS + + if det_tensor.shape[1] == nms_cols: + pass # already [N, 38] + elif det_tensor.shape[0] == nms_cols: + det_tensor = det_tensor.T + elif det_tensor.shape[1] == raw_cols: + pass # already [N, raw_cols] + elif det_tensor.shape[0] == raw_cols: + det_tensor = det_tensor.T + elif det_tensor.shape[0] < det_tensor.shape[1]: + det_tensor = det_tensor.T # likely [C, N] -> [N, C] + + return det_tensor, proto_tensor + + +def _instance_mask( + coeffs: np.ndarray, + protos: np.ndarray, + bbox: list[float], + scale: float, + pad: tuple[int, int], + input_size: int, + mask_threshold: float, +) -> tuple[str, int, int]: + """The detection's mask over its OWN box: the emitted (original-image) + bbox mapped back into the letterboxed input, then onto the prototype grid. + Only that window of ``coeffs @ protos`` is computed.""" + proto_size = protos.shape[-1] + proto_scale = proto_size / input_size + ox1, oy1, ox2, oy2 = bbox + px1 = max(0, int(np.floor((ox1 * scale + pad[0]) * proto_scale))) + py1 = max(0, int(np.floor((oy1 * scale + pad[1]) * proto_scale))) + px2 = min(proto_size, int(np.ceil((ox2 * scale + pad[0]) * proto_scale))) + py2 = min(proto_size, int(np.ceil((oy2 * scale + pad[1]) * proto_scale))) + crop_w = max(1, px2 - px1) + crop_h = max(1, py2 - py1) + window = protos[:, py1:py1 + crop_h, px1:px1 + crop_w] + h, w = window.shape[1], window.shape[2] + if h == 0 or w == 0: + binary = np.zeros((crop_h, crop_w), dtype=np.uint8) + else: + logits = coeffs.astype(np.float32) @ window.reshape(NUM_MASK_COEFFS, -1) + binary = (_sigmoid(logits.reshape(h, w)) > mask_threshold).astype(np.uint8) * 255 + crop_h, crop_w = h, w + return base64.b64encode(binary.tobytes()).decode("ascii"), crop_w, crop_h + + +def postprocess_yolo_seg( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """Postprocess YOLO-seg output to detections with per-instance masks. + + Handles: + 1. NMS-enabled export: [N, 6+32] with [x1,y1,x2,y2,conf,class_id,...coeffs] + 2. Raw export: [4+numClasses+32, N] standard YOLO transposed format + """ + labels = config.get("labels", COCO_80) + num_classes = config.get("numClasses", len(labels)) + input_size = int(config.get("inputSize", 640)) + mask_threshold = float(config.get("maskThreshold", 0.5)) + + det_tensor, proto_tensor = _find_tensors(predictions, num_classes) + + if det_tensor.shape[1] == 6 + NUM_MASK_COEFFS: + kept = decode_yolo11_nms_rows(det_tensor, config, orig_w, orig_h, scale, pad) + coeffs = det_tensor[:, 6:6 + NUM_MASK_COEFFS] + else: + # The raw decode reads [C, N] and may sigmoid its class rows in place: + # hand it its own copy so the coefficients below stay untouched. + kept = decode_raw_yolo_columns( + np.ascontiguousarray(det_tensor.T), config, orig_w, orig_h, scale, pad + ) + coeffs = det_tensor[:, 4 + num_classes:4 + num_classes + NUM_MASK_COEFFS] + + detections = [] + for index, det in kept: + mask_b64, mask_w, mask_h = _instance_mask( + coeffs[index], proto_tensor, det["bbox"], scale, pad, input_size, mask_threshold + ) + detections.append({**det, "mask": mask_b64, "maskWidth": mask_w, "maskHeight": mask_h}) + + return {"kind": "detections", "detections": detections} diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/yolo_split_dfl.py b/camstack_train/_vendor/pipeline_python/postprocessors/yolo_split_dfl.py new file mode 100644 index 0000000000000000000000000000000000000000..c249f58d344288e595b2caeddc894a8b994d78c1 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/yolo_split_dfl.py @@ -0,0 +1,255 @@ +"""YOLO split-head DFL postprocessor (Scrypted YOLOv9 TFLite / EdgeTPU). + +Scrypted's TFLite builds emit SIX tensors (3 strides × box+cls), not the +ONNX ``[1, 7, 2100]`` packed layout: + + box: [1, H·W, 4·reg_max] with ``reg_max=16`` (DFL distances l,t,r,b) + cls: [1, H·W, C] logits over person/vehicle/animal (C=3) + +Pairs are matched by shared spatial length ``N`` (1600/400/100 for 320²), not +by positional key order — TFLite output order has drifted between exports. +Coordinates are decoded in INPUT pixel space, then mapped back to the original +frame: stretch when ``preprocessMode == 'resize'`` (Scrypted default), else +letterbox undo via the pool's ``scale`` + ``pad``. + +Pure nested lists so CI's bare interpreter (no numpy) still pins the math — +same discipline as ``ssd.py``. A live ndarray from the pool is accepted via +``.tolist()``. +""" +from __future__ import annotations + +import math +from typing import Any + +REG_MAX = 16 + + +def _as_list(value: Any) -> Any: + return value.tolist() if hasattr(value, "tolist") else value + + +def _shape(value: Any) -> tuple[int, ...]: + dims: list[int] = [] + cur = value + while isinstance(cur, (list, tuple)): + dims.append(len(cur)) + if not cur: + break + cur = cur[0] + return tuple(dims) + + +def _sigmoid(x: float) -> float: + if x >= 0: + z = math.exp(-x) + return 1.0 / (1.0 + z) + z = math.exp(x) + return z / (1.0 + z) + + +def _softmax(xs: list[float]) -> list[float]: + m = max(xs) if xs else 0.0 + exps = [math.exp(x - m) for x in xs] + s = sum(exps) or 1.0 + return [e / s for e in exps] + + +def _dfl_distance(logits16: list[float]) -> float: + """Softmax over ``reg_max`` bins · project ``[0..reg_max)`` → distance in cells.""" + probs = _softmax(logits16) + return sum(i * p for i, p in enumerate(probs)) + + +def _iou(a: tuple[float, float, float, float], b: tuple[float, float, float, float]) -> float: + ax1, ay1, ax2, ay2 = a + bx1, by1, bx2, by2 = b + ix1, iy1 = max(ax1, bx1), max(ay1, by1) + ix2, iy2 = min(ax2, bx2), min(ay2, by2) + inter = max(0.0, ix2 - ix1) * max(0.0, iy2 - iy1) + if inter <= 0: + return 0.0 + area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1) + area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1) + union = area_a + area_b - inter + return inter / union if union > 0 else 0.0 + + +def _nms(boxes: list[dict], iou_threshold: float) -> list[dict]: + ordered = sorted(boxes, key=lambda b: b["score"], reverse=True) + kept: list[dict] = [] + suppressed: set[int] = set() + for i, box in enumerate(ordered): + if i in suppressed: + continue + kept.append(box) + for j in range(i + 1, len(ordered)): + if j in suppressed: + continue + if _iou(box["_xyxy"], ordered[j]["_xyxy"]) > iou_threshold: + suppressed.add(j) + return kept + + +def _squeeze_batch(tensor: Any) -> Any: + """Drop a leading batch dim of 1: [1, N, C] → [N, C].""" + shape = _shape(tensor) + if len(shape) == 3 and shape[0] == 1: + return tensor[0] + return tensor + + +def _pair_box_cls(predictions: dict) -> list[tuple[Any, Any]]: + """Group outputs into (box, cls) pairs by shared N (= H·W).""" + tensors = {str(k): _squeeze_batch(_as_list(v)) for k, v in predictions.items()} + boxes: dict[int, Any] = {} + classes: dict[int, Any] = {} + for tensor in tensors.values(): + shape = _shape(tensor) + if len(shape) != 2: + continue + n, width = shape + if width == 4 * REG_MAX: + boxes[n] = tensor + elif width > 0 and width != 4 * REG_MAX: + classes[n] = tensor + pairs: list[tuple[Any, Any]] = [] + for n, box in boxes.items(): + cls = classes.get(n) + if cls is None: + continue + pairs.append((box, cls)) + if not pairs: + raise ValueError( + "yolo-split-dfl: expected paired box[N,64] + cls[N,C] tensors; " + f"got shapes {[(_shape(_squeeze_batch(_as_list(v)))) for v in predictions.values()]}" + ) + # Largest N first (finest stride) — stable, matches Scrypted 1600→400→100. + pairs.sort(key=lambda p: _shape(p[0])[0], reverse=True) + return pairs + + +def _infer_stride(n: int, input_size: int) -> int: + """N = (input_size / stride)² → stride = input_size / sqrt(N).""" + side = int(round(math.sqrt(n))) + if side * side != n or side <= 0: + # Fall back to the canonical 320² strides by matching known Ns. + known = {1600: 8, 400: 16, 100: 32, (640 // 8) ** 2: 8} + if n in known: + return known[n] + raise ValueError(f"yolo-split-dfl: cannot infer grid side from N={n}") + stride = input_size // side + if stride * side != input_size: + # Non-divisible — still use the rounded stride from side. + stride = max(1, int(round(input_size / side))) + return stride + + +def _to_original( + x1: float, + y1: float, + x2: float, + y2: float, + *, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], + config: dict, +) -> tuple[float, float, float, float]: + mode = config.get("preprocessMode", "letterbox") + if mode == "resize": + in_w = float(config.get("inputWidth") or config.get("inputSize") or 320) + in_h = float(config.get("inputHeight") or config.get("inputSize") or 320) + if in_w <= 0 or in_h <= 0: + in_w = in_h = 320.0 + ox1 = x1 * orig_w / in_w + oy1 = y1 * orig_h / in_h + ox2 = x2 * orig_w / in_w + oy2 = y2 * orig_h / in_h + else: + ox1 = (x1 - pad[0]) / scale + oy1 = (y1 - pad[1]) / scale + ox2 = (x2 - pad[0]) / scale + oy2 = (y2 - pad[1]) / scale + return ( + max(0.0, min(float(orig_w), ox1)), + max(0.0, min(float(orig_h), oy1)), + max(0.0, min(float(orig_w), ox2)), + max(0.0, min(float(orig_h), oy2)), + ) + + +def postprocess_yolo_split_dfl( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + conf_threshold = float(config.get("confidence", 0)) + labels = config.get("labels") or ["person", "vehicle", "animal"] + nms_iou = float(config.get("nmsIouThreshold", 0.45)) + input_size = int(config.get("inputSize") or config.get("inputWidth") or 320) + + candidates: list[dict] = [] + for box_tensor, cls_tensor in _pair_box_cls(predictions): + n = _shape(box_tensor)[0] + stride = _infer_stride(n, input_size) + grid = int(round(math.sqrt(n))) + num_classes = _shape(cls_tensor)[1] + + for i in range(n): + # Class score first — skip cold cells before DFL. + logits = cls_tensor[i] + best_c = 0 + best_logit = float(logits[0]) + for c in range(1, num_classes): + v = float(logits[c]) + if v > best_logit: + best_logit = v + best_c = c + score = _sigmoid(best_logit) + if score < conf_threshold: + continue + + dist = box_tensor[i] + if len(dist) != 4 * REG_MAX: + continue + l = _dfl_distance([float(x) for x in dist[0:16]]) + t = _dfl_distance([float(x) for x in dist[16:32]]) + r = _dfl_distance([float(x) for x in dist[32:48]]) + b = _dfl_distance([float(x) for x in dist[48:64]]) + + gy, gx = i // grid, i % grid + # Ultralytics-style anchor centre at (gx + 0.5, gy + 0.5) * stride; + # distances are in cell units → multiply by stride for pixels. + cx = (gx + 0.5) * stride + cy = (gy + 0.5) * stride + x1 = cx - l * stride + y1 = cy - t * stride + x2 = cx + r * stride + y2 = cy + b * stride + + ox1, oy1, ox2, oy2 = _to_original( + x1, y1, x2, y2, + orig_w=orig_w, orig_h=orig_h, scale=scale, pad=pad, config=config, + ) + if ox2 <= ox1 or oy2 <= oy1: + continue + + label = labels[best_c] if best_c < len(labels) else str(best_c) + candidates.append( + { + "class": label, + "score": round(score, 4), + "bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)], + "_xyxy": (ox1, oy1, ox2, oy2), + } + ) + + kept = _nms(candidates, nms_iou) + detections = [ + {"class": d["class"], "score": d["score"], "bbox": d["bbox"]} for d in kept + ] + return {"kind": "detections", "detections": detections} diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/yolonas.py b/camstack_train/_vendor/pipeline_python/postprocessors/yolonas.py new file mode 100644 index 0000000000000000000000000000000000000000..5eb674448eed889ea00e69bedbe0f5cbc20cc056 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/yolonas.py @@ -0,0 +1,67 @@ +"""YOLO-NAS postprocessor — Frigate-style flat batched-NMS ONNX export. + +Frigate+ ``yolonas`` models (and any super-gradients ``model.export()`` with +fused NMS in FLAT format) emit ONE output tensor of shape ``[N, 7]`` where each +row is ``(batch_index, x1, y1, x2, y2, score, class_id)``: + + - coordinates are in INPUT pixel space (0..inputSize), post-NMS, + - GPU exports pad unused slots with a NEGATIVE class id (skip them), + - class ids index the model's own label map (Frigate+ label set or COCO-80). + +Output matches the yolo postprocessor: +``{"kind": "detections", "detections": [{"class", "score", "bbox": [x1,y1,x2,y2]}]}`` +with pixel bboxes re-projected to ORIGINAL frame coordinates (letterbox undone +via the pool-provided ``scale`` + ``pad``). +""" +from __future__ import annotations + +import numpy as np + +from .yolo import COCO_80 + + +def postprocess_yolonas( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + conf_threshold = float(config.get("confidence", 0)) + labels = config.get("labels", COCO_80) + + output = np.asarray(list(predictions.values())[0]) + # Accept [1, N, 7] batched exports by squeezing the batch axis. + if output.ndim == 3: + output = output.reshape(-1, output.shape[-1]) + if output.ndim != 2 or output.shape[-1] != 7: + raise ValueError( + f"yolonas postprocessor expects a flat [N, 7] output, got shape {output.shape}" + ) + + detections = [] + for row in output: + _batch_idx, x1, y1, x2, y2, score, class_id = row + score = float(score) + class_id = int(class_id) + if class_id < 0: + continue + if score < conf_threshold: + continue + + ox1 = max(0.0, min(float(orig_w), (float(x1) - pad[0]) / scale)) + oy1 = max(0.0, min(float(orig_h), (float(y1) - pad[1]) / scale)) + ox2 = max(0.0, min(float(orig_w), (float(x2) - pad[0]) / scale)) + oy2 = max(0.0, min(float(orig_h), (float(y2) - pad[1]) / scale)) + + label = labels[class_id] if class_id < len(labels) else str(class_id) + detections.append( + { + "class": label, + "score": round(score, 4), + "bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)], + } + ) + + return {"kind": "detections", "detections": detections} diff --git a/camstack_train/_vendor/pipeline_python/postprocessors/yunet.py b/camstack_train/_vendor/pipeline_python/postprocessors/yunet.py new file mode 100644 index 0000000000000000000000000000000000000000..0ab5503d6e694d73c26a10b695706b0750e43290 --- /dev/null +++ b/camstack_train/_vendor/pipeline_python/postprocessors/yunet.py @@ -0,0 +1,275 @@ +"""YuNet face detection postprocessor (OpenCV Zoo `face_detection_yunet`, 2023mar). + +Used by the `yunet-2023mar` catalog entry on the `face-detection` step through a +per-MODEL `postprocessor: 'yunet'` override (the step default stays `scrfd`). + +Model contract, as published by `scripts/build-yunet-model.py`: + + input "image" [1, 3, 640, 640] RGB float in [0,1] — the pool's plain + `/255` tensor. Upstream YuNet wants BGR + 0..255; `Mul(255)` + `Gather([2,1,0])` + are BAKED into the graph ahead of the + first Conv, identically in all three + formats, so no pool normalisation runs + (the entry declares none). + outputs cls_{8,16,32} [1, N, 1] classification probability + obj_{8,16,32} [1, N, 1] objectness probability + bbox_{8,16,32} [1, N, 4] (dx, dy, log w, log h), stride units + kps_{8,16,32} [1, N, 10] 5 × (dx, dy), stride units + +Why it cannot share `scrfd.py` (measured 2026-09-26: the SCRFD decoder finds 0 +faces on 413 person crops with YuNet's outputs): + +1. ONE prior per grid cell, not two; anchor-free — the prior is the cell's + top-left corner, `(col, row) * stride`, row-major, `N = ceil(640/s)^2`. +2. The box is centre + size, with `exp()` on width and height — SCRFD's is four + distances from the anchor. +3. The score is `sqrt(cls * obj)` over TWO tensors, both clamped to [0,1]. + +This is a port of OpenCV's `FaceDetectorYN::postProcess` +(`modules/objdetect/src/face_detect.cpp`). + +Landmarks come out in YuNet's order — right eye, left eye, nose tip, right +mouth corner, left mouth corner, where right/left are the SUBJECT'S. That is +image-left eye first, the same order SCRFD emits and `face-align.ts` expects +(ARCFACE_TEMPLATE_112), so they are passed through unchanged. Measured on 196 +COCO faces: index 0 is the image-left eye except on 3 profile faces, the same +rate as SCRFD's 2. + +The per-candidate arithmetic (`prior_cell`, `decode_prior`, `to_source`, `nms`) +is plain Python so the layout, decode and landmark-order tests run on CI's bare +interpreter; numpy only vectorises the score threshold over the 8400 priors. + +Output matches the scrfd postprocessor: +``{"kind": "detections", "detections": [{"class": "face", "score", "bbox": [x1,y1,x2,y2], "landmarks"}]}`` +in ORIGINAL (pre-letterbox) pixel coordinates. +""" +from __future__ import annotations + +import math +import sys +import time + +import numpy as np + +STRIDES: tuple[int, ...] = (8, 16, 32) +NUM_LANDMARKS = 5 +DEFAULT_NMS_IOU = 0.45 + +# At most this many candidates, best first, reach the NMS. OpenCV's +# FaceDetectorYN caps at 5000 (`top_k`), but its NMS is C++; `nms` below is +# pure Python and O(n·kept), measured on this Mac at 28 ms for 500 disjoint +# candidates, 113 ms for 1000 and 2.8 s for 5000 — an unbounded frame (every +# prior passing a low floor: 8400) would hold a pool worker for ~8 s. Real +# frames are far below the cap: on the spike's 413 person crops the most +# candidates any frame produced was 38 at the step's 0.5 floor and 110 at 0.1. +TOP_K = 750 +# A frame that hits the cap is reported, at most once per this interval, on +# stderr (the pool's log): dropped work is never silent, and a pathological +# model/threshold must not flood the log either. +TOP_K_LOG_INTERVAL_S = 60.0 +_last_top_k_log = 0.0 + + +def grid_cols(stride: int, input_size: int) -> int: + """Priors per row (and per column — the input is square) at `stride`.""" + return int(math.ceil(input_size / stride)) + + +def prior_cell(index: int, cols: int) -> tuple[int, int]: + """`(row, col)` of prior `index` — row-major, one prior per cell.""" + return divmod(index, cols) + + +def score_of(cls: float, obj: float) -> float: + """YuNet's score: geometric mean of the two probabilities, each clamped.""" + return math.sqrt(min(max(cls, 0.0), 1.0) * min(max(obj, 0.0), 1.0)) + + +def decode_prior( + stride: int, + row: int, + col: int, + bbox: "list[float] | tuple[float, ...]", + kps: "list[float] | tuple[float, ...]", +) -> tuple[tuple[float, float, float, float], list[tuple[float, float]]]: + """One prior → `(x1, y1, x2, y2)` and 5 landmark points, in NETWORK space.""" + cx = (col + bbox[0]) * stride + cy = (row + bbox[1]) * stride + w = math.exp(bbox[2]) * stride + h = math.exp(bbox[3]) * stride + box = (cx - w / 2.0, cy - h / 2.0, cx + w / 2.0, cy + h / 2.0) + points = [ + ((kps[2 * p] + col) * stride, (kps[2 * p + 1] + row) * stride) + for p in range(NUM_LANDMARKS) + ] + return box, points + + +def to_source(x: float, y: float, scale: float, pad: tuple[int, int]) -> tuple[float, float]: + """Undo the pool's letterbox: network pixels → original frame pixels.""" + return (x - pad[0]) / scale, (y - pad[1]) / scale + + +def _iou(a: tuple, b: tuple) -> float: + ix = max(0.0, min(a[2], b[2]) - max(a[0], b[0])) + iy = max(0.0, min(a[3], b[3]) - max(a[1], b[1])) + inter = ix * iy + if inter == 0: + return 0.0 + union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter + return inter / union if union > 0 else 0.0 + + +def cap_top_k(candidates: list[dict], k: int = TOP_K) -> tuple[list[dict], int]: + """The `k` best candidates by score, and how many were dropped.""" + if len(candidates) <= k: + return candidates, 0 + best = sorted(candidates, key=lambda c: c["score"], reverse=True) + return best[:k], len(candidates) - k + + +def _log_top_k_drop(dropped: int, total: int) -> None: + global _last_top_k_log + now = time.monotonic() + if now - _last_top_k_log < TOP_K_LOG_INTERVAL_S: + return + _last_top_k_log = now + sys.stderr.write( + f"yunet: {total} candidates above the confidence floor, kept the best {TOP_K} " + f"for NMS and dropped {dropped} (logged at most once per " + f"{TOP_K_LOG_INTERVAL_S:.0f}s)\n" + ) + sys.stderr.flush() + + +def nms(candidates: list[dict], iou_threshold: float) -> list[dict]: + """Greedy NMS on `_xyxy`, highest score first.""" + kept: list[dict] = [] + for cand in sorted(candidates, key=lambda c: c["score"], reverse=True): + if all(_iou(cand["_xyxy"], k["_xyxy"]) <= iou_threshold for k in kept): + kept.append(cand) + return kept + + +_OUTPUT_WIDTHS: tuple[tuple[str, int], ...] = ( + ("cls", 1), ("obj", 1), ("bbox", 4), ("kps", 2 * NUM_LANDMARKS), +) + + +def missing_outputs(output_names) -> list[str]: + """The named outputs this decoder needs that the model did not produce. + + Matched by NAME only, with no shape fallback: `cls_*` and `obj_*` have + identical shapes, so a fallback could only guess. Every published format + names its outputs (the CoreML conversion names them explicitly, the + OpenVINO IR has its internal aliases cleared).""" + present = set(output_names) + return [ + f"{kind}_{stride}" for stride in STRIDES for kind, _ in _OUTPUT_WIDTHS + if f"{kind}_{stride}" not in present + ] + + +def value_count(value) -> int: + """Number of scalars in an ndarray-like (`.size`) or a nested sequence. + Pure Python, so the size gate below needs no numpy.""" + size = getattr(value, "size", None) + if isinstance(size, int): + return size + if isinstance(value, (list, tuple)): + return sum(value_count(v) for v in value) + return 1 + + +def check_outputs(predictions: dict, input_size: int) -> None: + """RAISE unless all twelve outputs are present and sized for `input_size`. + + Returning `[]` instead would read as "no faces" forever with no line + saying why — which is exactly what the SCRFD decoder did on this graph. A + size mismatch means the model's input is not `inputSize` (e.g. a 320 + export under a 640 catalog entry): decoding it would put every prior in the + wrong cell.""" + missing = missing_outputs(predictions.keys()) + if missing: + raise ValueError( + f"yunet: outputs {missing} missing (model outputs: {sorted(predictions)})" + ) + for stride in STRIDES: + priors = grid_cols(stride, input_size) ** 2 + for kind, width in _OUTPUT_WIDTHS: + key = f"{kind}_{stride}" + got = value_count(predictions[key]) + if got != priors * width: + raise ValueError( + f"yunet: output '{key}' has {got} values, expected {priors * width} " + f"({priors} priors × {width}) — inputSize {input_size} does not match the model" + ) + + +def _stride_tensors(predictions: dict, stride: int) -> dict: + """The four flattened tensors of one stride (sizes already checked).""" + return { + kind: np.asarray(predictions[f"{kind}_{stride}"], dtype=np.float32).reshape(-1) + for kind, _ in _OUTPUT_WIDTHS + } + + +def postprocess_yunet( + predictions: dict, + config: dict, + orig_w: int, + orig_h: int, + scale: float, + pad: tuple[int, int], +) -> dict: + """Postprocess YuNet output to structured face detections.""" + conf_threshold = float(config.get("confidence", 0.5) or 0.5) + input_size = int(config.get("inputSize", 640)) + nms_iou = float(config.get("nmsIouThreshold", DEFAULT_NMS_IOU)) + + check_outputs(predictions, input_size) + + candidates: list[dict] = [] + for stride in STRIDES: + cols = grid_cols(stride, input_size) + t = _stride_tensors(predictions, stride) + bbox = t["bbox"].reshape(-1, 4) + kps = t["kps"].reshape(-1, 2 * NUM_LANDMARKS) + # numpy only PRE-filters, conservatively: sqrt(cls·obj) <= max(cls, obj), + # so a prior below the threshold on both tensors cannot pass. The + # decision itself is `score_of`, per prior. + maybe = np.flatnonzero(np.maximum(t["cls"], t["obj"]) >= conf_threshold) + for index in maybe.tolist(): + score = score_of(float(t["cls"][index]), float(t["obj"][index])) + if score < conf_threshold: + continue + row, col = prior_cell(index, cols) + (x1, y1, x2, y2), points = decode_prior( + stride, row, col, bbox[index].tolist(), kps[index].tolist() + ) + sx1, sy1 = to_source(x1, y1, scale, pad) + sx2, sy2 = to_source(x2, y2, scale, pad) + ox1, oy1 = max(0.0, min(orig_w, sx1)), max(0.0, min(orig_h, sy1)) + ox2, oy2 = max(0.0, min(orig_w, sx2)), max(0.0, min(orig_h, sy2)) + if (ox2 - ox1) < 1 or (oy2 - oy1) < 1: + continue + landmarks = [] + for px, py in points: + lx, ly = to_source(px, py, scale, pad) + landmarks.append({"x": round(lx, 1), "y": round(ly, 1)}) + candidates.append({ + "class": "face", + "score": round(score, 4), + "bbox": [round(ox1, 1), round(oy1, 1), round(ox2, 1), round(oy2, 1)], + "landmarks": landmarks, + "_xyxy": (ox1, oy1, ox2, oy2), + }) + + capped, dropped = cap_top_k(candidates) + if dropped: + _log_top_k_drop(dropped, len(candidates)) + kept = nms(capped, nms_iou) + for det in kept: + del det["_xyxy"] + return {"kind": "detections", "detections": kept} diff --git a/camstack_train/cli.py b/camstack_train/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..32efd236be67b491d184c455f128838e23b07124 --- /dev/null +++ b/camstack_train/cli.py @@ -0,0 +1,70 @@ +"""``python -m camstack_train run|dataset``. + +``run``: one job. The hub's provider starts the container with +``--job /data/job.json --dataset /data/dataset.tar --out /data/out --cache /cache``. +``dataset``: the export → YOLO layout only, for training by hand. + +Exit codes: 0 ok, 64 bad job.json, 65 dataset unusable, 70 internal. +""" +from __future__ import annotations + +import argparse +import tempfile +from pathlib import Path +from typing import Optional, Sequence + +from .dataset import extract_archive, frames_from_manifest, write_layout +from .errors import EXIT_BAD_SPEC, EXIT_OK, TrainerError +from .events import Emitter +from .manifest import failed_result, write_result +from .run import Deps, Paths, run_job +from .spec import job_key_of, load_job + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="camstack_train", description="CamStack detector trainer") + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run", help="run one training job") + run.add_argument("--job", type=Path, required=True) + run.add_argument("--dataset", type=Path, required=True) + run.add_argument("--out", type=Path, required=True) + run.add_argument("--cache", type=Path, required=True) + run.add_argument("--work", type=Path, default=None, help="scratch dir (default: a temp dir)") + ds = sub.add_parser("dataset", help="convert a hub export to a YOLO layout (all frames → train)") + ds.add_argument("--export", type=Path, required=True) + ds.add_argument("--out", type=Path, required=True) + return parser + + +def _run(args: argparse.Namespace, deps: Optional[Deps]) -> int: + emitter = deps.emitter if deps is not None else Emitter() + try: + job = load_job(args.job) + except TrainerError as exc: + emitter.error(exc.stage, exc.code, exc.message) + digest = write_result(args.out / "result.json", failed_result(job_key_of(args.job), exc.as_dict())) + emitter.done("failed", digest) + return EXIT_BAD_SPEC + work = args.work or Path(tempfile.mkdtemp(prefix="camstack-train-")) + paths = Paths(job=args.job, dataset=args.dataset, out=args.out, cache=args.cache, work=work) + return run_job(job, paths, deps) + + +def _dataset(args: argparse.Namespace) -> int: + source = args.out / "export" + manifest = extract_archive(args.export, source) + frames = [f for f in frames_from_manifest(manifest) if f.trainable] + counts = write_layout({"train": frames}, source, args.out / "yolo") + print(f"{counts.frames.get('train', 0)} frames, {counts.boxes.get('train', 0)} boxes → {args.out / 'yolo'}") + return EXIT_OK + + +def main(argv: Optional[Sequence[str]] = None, deps: Optional[Deps] = None) -> int: + args = _parser().parse_args(argv) + if args.command == "run": + return _run(args, deps) + try: + return _dataset(args) + except TrainerError as exc: + print(f"{exc.code}: {exc.message}") + return exc.exit_code diff --git a/camstack_train/dataset.py b/camstack_train/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..3c52db1a0ebfc09fea761067fcae219507565308 --- /dev/null +++ b/camstack_train/dataset.py @@ -0,0 +1,228 @@ +"""The hub's annotated export → an Ultralytics YOLO layout. + +Input: the retrain archive (``annotations.json`` first, then ``frames/…`` JPEGs; +schema ``camstack.retrain-export/v2`` or ``/v3``), exactly what +``pipelineAnalytics.writeRetrainArchive`` streams. + +What becomes a label (COCO-80 ids, the head the fleet's root keeps): + +- ``subject`` of macro ``person`` → ``person`` (0). +- ``subject`` of macro ``vehicle`` / ``animal`` → its COCO class, taken from the + annotation's ``label`` (``car``, ``dog``…) when that label is a COCO class of + the same macro. A vehicle or animal box WITHOUT such a label cannot be given + a COCO class honestly (training "dog" on a cat teaches the wrong species), so + its frame is left out of training and counted as ``unmappable``. Evaluation + is per macro class and still uses it. +- ``model_error`` boxes are never positives: the region is learned as + background, which is what marking a phantom asks for. +- ``package`` / ``face`` / ``plate`` boxes are not root-detector classes and are + ignored (no label line). + +Which frames TRAIN: + +- every ``complete`` frame ("every subject is boxed"), including one with no + subject at all — a hard negative; +- an INCOMPLETE frame only when it carries at least one annotation (a subject + or a model error): the operator picked it to teach that box. YOLO has no + "ignore region", so an unboxed subject on such a frame IS learned as + background; restricting incomplete frames to annotated ones is the bound on + that cost. Incomplete frames are never evaluated. + +Files are renamed to neutral ``000001.jpg`` names: archive paths carry device +and track ids, and nothing the container writes or logs names a frame. +""" +from __future__ import annotations + +import hashlib +import json +import shutil +import tarfile +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import Iterable, Optional + +from ._vendor import ensure_pipeline_on_path +from ._vendor.camstack_eval.classes import COCO_TO_MACRO +from .errors import dataset_error + +MANIFEST = "annotations.json" +ACCEPTED_SCHEMAS = ("camstack.retrain-export/v2", "camstack.retrain-export/v3") +IGNORED_MACROS = ("package", "face", "plate") + + +def coco_80() -> list[str]: + ensure_pipeline_on_path() + from postprocessors.yolo import COCO_80 # the pool's own label list + + return list(COCO_80) + + +@dataclass(frozen=True) +class YoloBox: + cls_index: int + cx: float + cy: float + w: float + h: float + + def line(self) -> str: + return f"{self.cls_index} {self.cx:.6f} {self.cy:.6f} {self.w:.6f} {self.h:.6f}" + + +@dataclass(frozen=True) +class Frame: + frame_id: str + path: str + device_id: int + captured_at: Optional[int] + complete: bool + labels: tuple[YoloBox, ...] + subjects: int + model_errors: int + unmappable: int + + @property + def trainable(self) -> bool: + if self.unmappable > 0: + return False + return self.complete or (self.subjects + self.model_errors) > 0 + + +@dataclass +class LayoutCounts: + frames: dict[str, int] = field(default_factory=dict) + boxes: dict[str, int] = field(default_factory=dict) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_archive(path: Path, expected_sha256: str) -> None: + if not path.is_file(): + raise dataset_error("dataset-missing", "dataset.tar not found") + actual = sha256_file(path) + if actual != expected_sha256: + raise dataset_error("dataset-sha256", "dataset.tar does not match the job's sha256") + + +def _safe_member(name: str) -> bool: + pure = PurePosixPath(name) + if pure.is_absolute() or ".." in pure.parts: + return False + return name == MANIFEST or (pure.parts[:1] == ("frames",) and pure.suffix.lower() == ".jpg") + + +def extract_archive(path: Path, dest: Path) -> dict: + """Extract the manifest and the frame JPEGs only; return the manifest.""" + dest.mkdir(parents=True, exist_ok=True) + try: + with tarfile.open(path) as tar: + for member in tar: + if not member.isfile() or not _safe_member(member.name): + continue + target = dest / member.name + target.parent.mkdir(parents=True, exist_ok=True) + source = tar.extractfile(member) + if source is None: + continue + with target.open("wb") as out: + shutil.copyfileobj(source, out) + except tarfile.TarError as exc: + raise dataset_error("dataset-unreadable", "dataset.tar is not a readable tar") from exc + manifest_path = dest / MANIFEST + if not manifest_path.is_file(): + raise dataset_error("dataset-no-manifest", "dataset.tar has no annotations.json") + manifest = json.loads(manifest_path.read_text("utf-8")) + if manifest.get("schema") not in ACCEPTED_SCHEMAS: + raise dataset_error("dataset-schema", f"export schema {manifest.get('schema')!r} is not supported") + return manifest + + +def _clamped(bbox: dict) -> Optional[tuple[float, float, float, float]]: + x1 = max(0.0, float(bbox["x"])) + y1 = max(0.0, float(bbox["y"])) + x2 = min(1.0, float(bbox["x"]) + float(bbox["w"])) + y2 = min(1.0, float(bbox["y"]) + float(bbox["h"])) + if x2 - x1 <= 1e-4 or y2 - y1 <= 1e-4: + return None + return (x1 + x2) / 2, (y1 + y2) / 2, x2 - x1, y2 - y1 + + +def coco_index_for(macro: str, label: Optional[str], names: list[str]) -> Optional[int]: + """The COCO-80 index a subject trains as, or ``None`` when none is honest.""" + if macro == "person": + return names.index("person") + wanted = (label or "").strip().lower() + if wanted in names and COCO_TO_MACRO.get(wanted) == macro: + return names.index(wanted) + return None + + +def frame_from_entry(entry: dict, names: list[str]) -> Frame: + labels: list[YoloBox] = [] + subjects = model_errors = unmappable = 0 + for annotation in entry.get("annotations", []): + macro = annotation.get("macroClass") + if annotation.get("kind") == "model_error": + model_errors += 1 + continue + if annotation.get("kind") != "subject" or macro in IGNORED_MACROS: + continue + subjects += 1 + index = coco_index_for(str(macro), annotation.get("label"), names) + if index is None: + unmappable += 1 + continue + box = _clamped(annotation["bbox"]) + if box is not None: + labels.append(YoloBox(index, *box)) + captured = entry.get("capturedAt") + return Frame( + frame_id=str(entry["frameId"]), + path=str(entry["path"]), + device_id=int(entry["deviceId"]), + captured_at=int(captured) if isinstance(captured, (int, float)) else None, + complete=entry.get("complete") is True, + labels=tuple(labels), + subjects=subjects, + model_errors=model_errors, + unmappable=unmappable, + ) + + +def frames_from_manifest(manifest: dict, names: Optional[list[str]] = None) -> list[Frame]: + names = names if names is not None else coco_80() + return [frame_from_entry(e, names) for e in manifest.get("frames", [])] + + +def write_split(frames: Iterable[Frame], source_root: Path, out_root: Path, split: str) -> tuple[int, int]: + """Copy one split's frames + YOLO label files; returns (frames, boxes).""" + images = out_root / "images" / split + labels = out_root / "labels" / split + images.mkdir(parents=True, exist_ok=True) + labels.mkdir(parents=True, exist_ok=True) + count = boxes = 0 + for frame in frames: + source = source_root / frame.path + if not source.is_file(): + continue + count += 1 + stem = f"{count:06d}" + shutil.copyfile(source, images / f"{stem}.jpg") + (labels / f"{stem}.txt").write_text("".join(b.line() + "\n" for b in frame.labels), "utf-8") + boxes += len(frame.labels) + return count, boxes + + +def write_layout(splits: dict[str, list[Frame]], source_root: Path, out_root: Path) -> LayoutCounts: + counts = LayoutCounts() + for split, frames in splits.items(): + n, boxes = write_split(frames, source_root, out_root, split) + counts.frames[split] = n + counts.boxes[split] = boxes + return counts diff --git a/camstack_train/errors.py b/camstack_train/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..f814d5bdcaf9fae09f8607169620e2220893599a --- /dev/null +++ b/camstack_train/errors.py @@ -0,0 +1,34 @@ +"""The trainer's one failure type and its exit codes.""" +from __future__ import annotations + +#: Process exit codes the hub reads. +EXIT_OK = 0 +EXIT_BAD_SPEC = 64 +EXIT_DATASET = 65 +EXIT_INTERNAL = 70 + + +class TrainerError(Exception): + """A failure with a stage, a stable code and an exit code. + + ``message`` is shown to the operator: it never names a frame path or a + camera, only counts and codes. + """ + + def __init__(self, stage: str, code: str, message: str, exit_code: int) -> None: + super().__init__(message) + self.stage = stage + self.code = code + self.message = message + self.exit_code = exit_code + + def as_dict(self) -> dict[str, str]: + return {"stage": self.stage, "code": self.code, "message": self.message} + + +def spec_error(code: str, message: str) -> TrainerError: + return TrainerError("prepare", code, message, EXIT_BAD_SPEC) + + +def dataset_error(code: str, message: str) -> TrainerError: + return TrainerError("prepare", code, message, EXIT_DATASET) diff --git a/camstack_train/evaluate.py b/camstack_train/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..449219f4f2e05a7e90ca57d326a88687a01d4f9b --- /dev/null +++ b/camstack_train/evaluate.py @@ -0,0 +1,117 @@ +"""Baseline vs candidate on the holdout, through the hub's own evaluation harness. + +The harness (``_vendor/camstack_eval``, byte-identical to +``scripts/eval/camstack_eval``) runs both models with the inference pool's +letterbox and decode, scores P/R/AP50 per macro class, camera, day/night and +small subjects, and proposes the candidate's floors by the D739 rule against +the baseline's shipped floors. Only COMPLETE holdout frames are ground truth. + +The baseline is the catalog ONNX the hub names (``baseline.onnxUrl``), fetched +once into the cache volume. The candidate is the exported ONNX; the INT8 IR is +evaluated separately, as its own model with its own floors. +""" +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Optional, Sequence + +from ._vendor import ensure_pipeline_on_path +from ._vendor.camstack_eval.compare import compare_models +from ._vendor.camstack_eval.dataset import EvalFrame, load_export +from ._vendor.camstack_eval.floors import FloorPolicy +from ._vendor.camstack_eval.metrics import Detection +from .replay import Fetch +from .spec import JobSpec + +Detector = Callable[[bytes], Sequence[Detection]] +DetectorFactory = Callable[[str, int], Detector] + + +def harness_detector(model_path: str, input_size: int) -> Detector: + ensure_pipeline_on_path() + from ._vendor.camstack_eval.pipeline import make_detector + + return make_detector(model_path, input_size).detect + + +def policy_of(job: JobSpec) -> FloorPolicy: + return FloorPolicy(min_support=job.floors.min_support, clamp=job.floors.clamp) + + +def baseline_model(job: JobSpec, cache: Path, fetch: Fetch) -> Path: + digest = hashlib.sha256(job.baseline.onnx_url.encode()).hexdigest()[:16] + target = cache / "baselines" / f"{digest}.onnx" + if not target.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + partial = target.with_suffix(".part") + partial.write_bytes(fetch(job.baseline.onnx_url)) + partial.rename(target) + return target + + +def holdout_frames(dataset_root: Path, frame_ids: set[str]) -> list[EvalFrame]: + return [f for f in load_export(str(dataset_root), complete_only=True) if f.truth.frame_id in frame_ids] + + +def map50(report: Optional[dict]) -> Optional[float]: + """Mean AP50 of the candidate over the classes that have one.""" + if report is None: + return None + values = [c["candidate"]["ap50"] for c in report.get("classes", {}).values() if c["candidate"]["ap50"] is not None] + return round(sum(values) / len(values), 4) if values else None + + +@dataclass(frozen=True) +class Evaluation: + candidate: Optional[dict] + candidate_int8: Optional[dict] + holdout_frames: int + + +def evaluate( + job: JobSpec, + frames: Sequence[EvalFrame], + baseline_path: Path, + candidate_onnx: Path, + candidate_int8: Optional[Path], + make: DetectorFactory = harness_detector, +) -> Evaluation: + """Each report holds ``baseline`` and ``candidate`` per class (the harness's shape).""" + if not frames: + return Evaluation(None, None, 0) + baseline = make(str(baseline_path), job.baseline.imgsz) + policy = policy_of(job) + floors = dict(job.baseline.floors) + candidate = compare_models(frames, baseline, make(str(candidate_onnx), job.base.imgsz), floors, policy=policy) + int8 = None + if candidate_int8 is not None: + int8 = compare_models(frames, baseline, make(str(candidate_int8), job.base.imgsz), floors, policy=policy) + return Evaluation(candidate, int8, len(frames)) + + +def split_reports(evaluation: Evaluation) -> dict: + """The result.json ``evaluation`` block: baseline / candidate / candidateInt8.""" + + def side(report: Optional[dict], key: str) -> Optional[dict]: + if report is None: + return None + return { + "frames": report["frames"], + "classes": {cls: {"positives": c["positives"], **c[key]} for cls, c in report["classes"].items()}, + } + + def deltas(report: Optional[dict]) -> Optional[dict]: + if report is None: + return None + return {cls: c["recallDeltaByCamera"] for cls, c in report["classes"].items()} + + return { + "holdoutFrames": evaluation.holdout_frames, + "baseline": side(evaluation.candidate, "baseline"), + "candidate": side(evaluation.candidate, "candidate"), + "candidateInt8": side(evaluation.candidate_int8, "candidate"), + "recallDeltaByCamera": deltas(evaluation.candidate), + "map50": {"candidate": map50(evaluation.candidate), "candidateInt8": map50(evaluation.candidate_int8)}, + } diff --git a/camstack_train/events.py b/camstack_train/events.py new file mode 100644 index 0000000000000000000000000000000000000000..cc1344599fb9b246726090394c58ac62adcfcafe --- /dev/null +++ b/camstack_train/events.py @@ -0,0 +1,87 @@ +"""``CAMSTACK_EVENT {json}`` progress lines on stdout. + +The hub reads the provider's log and parses these lines; it never needs to +understand anything else the container prints. Every event carries ``v`` (the +event format version, 1), ``ts`` (epoch ms) and ``type``. One event per line, +compact JSON, flushed immediately. +""" +from __future__ import annotations + +import json +import sys +import time +from typing import Any, Callable, Optional, TextIO + +PREFIX = "CAMSTACK_EVENT " +EVENT_VERSION = 1 + +STAGES = ("prepare", "replay", "train", "export", "eval", "package") + + +class Emitter: + def __init__( + self, + stream: Optional[TextIO] = None, + clock: Callable[[], float] = time.time, + ) -> None: + self._stream = stream + self._clock = clock + + def emit(self, event_type: str, **fields: Any) -> dict[str, Any]: + event: dict[str, Any] = { + "v": EVENT_VERSION, + "ts": int(self._clock() * 1000), + "type": event_type, + **fields, + } + stream = self._stream if self._stream is not None else sys.stdout + stream.write(PREFIX + json.dumps(event, separators=(",", ":"), sort_keys=False) + "\n") + stream.flush() + return event + + def stage(self, stage: str) -> None: + if stage not in STAGES: + raise ValueError(f"unknown stage {stage!r}") + self.emit("stage", stage=stage) + + def epoch( + self, + epoch: int, + epochs: int, + box_loss: Optional[float], + map50: Optional[float], + map50_95: Optional[float], + eta_sec: Optional[float], + ) -> None: + self.emit( + "epoch", + epoch=epoch, + epochs=epochs, + boxLoss=box_loss, + map50=map50, + map50_95=map50_95, + etaSec=eta_sec, + ) + + def metric(self, name: str, value: float) -> None: + self.emit("metric", name=name, value=value) + + def artifact(self, path: str, size: int, sha256: str) -> None: + self.emit("artifact", path=path, bytes=size, sha256=sha256) + + def error(self, stage: str, code: str, message: str) -> None: + self.emit("error", stage=stage, code=code, message=message) + + def done(self, status: str, result_sha256: Optional[str]) -> None: + self.emit("done", status=status, resultSha256=result_sha256) + + +def parse_line(line: str) -> Optional[dict[str, Any]]: + """The event on a log line, or ``None`` for any other line.""" + if not line.startswith(PREFIX): + return None + try: + value = json.loads(line[len(PREFIX):]) + except json.JSONDecodeError: + return None + return value if isinstance(value, dict) else None diff --git a/camstack_train/export.py b/camstack_train/export.py new file mode 100644 index 0000000000000000000000000000000000000000..157d7c441219faffca61a6d2392fc47832c7e2d1 --- /dev/null +++ b/camstack_train/export.py @@ -0,0 +1,142 @@ +"""best.pt → ONNX → OpenVINO fp16 / int8 → CoreML fp16, into the staging dir. + +Every artefact is named after the catalog id, because a node's models +directory is FLAT and a format's file is found by its URL's basename: + + .pt .onnx -fp16.xml/.bin -int8.xml/.bin .mlpackage/ + +ONNX is required (every other format is built from it, and the evaluation runs +it): its failure fails the job. Each other format is isolated — a failure +drops THAT format with its reason (``exportsDropped``) and the job goes on; +CoreML on Linux is the expected casualty, and a Mac node can still convert the +ONNX later through Model Studio. +""" +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Protocol + +from .errors import EXIT_INTERNAL, TrainerError + +#: Characters of a failed step's output kept as its reason. +REASON_CHARS = 400 + + +class Toolchain(Protocol): + def onnx(self, pt: Path, out: Path, imgsz: int) -> None: ... + def openvino(self, onnx: Path, xml: Path, fp16: bool) -> None: ... + def openvino_int8(self, onnx: Path, xml: Path, calib: list[Path], imgsz: int) -> None: ... + def coreml(self, pt: Path, out: Path, imgsz: int) -> None: ... + + +class StepFailed(Exception): + pass + + +class SubprocessToolchain: + """Runs each step in a child (``camstack_train._steps``).""" + + def _run(self, step: str, payload: dict) -> None: + proc = subprocess.run( + [sys.executable, "-m", "camstack_train._steps", step, json.dumps(payload)], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + tail = (proc.stderr or proc.stdout or "").strip()[-REASON_CHARS:] + raise StepFailed(f"{step} exited {proc.returncode}: {tail}") + + def onnx(self, pt: Path, out: Path, imgsz: int) -> None: + self._run("onnx", {"pt": str(pt), "out": str(out), "imgsz": imgsz}) + + def openvino(self, onnx: Path, xml: Path, fp16: bool) -> None: + self._run("openvino", {"onnx": str(onnx), "xml": str(xml), "fp16": fp16}) + + def openvino_int8(self, onnx: Path, xml: Path, calib: list[Path], imgsz: int) -> None: + self._run("openvino-int8", {"onnx": str(onnx), "xml": str(xml), "calib": [str(p) for p in calib], "imgsz": imgsz}) + + def coreml(self, pt: Path, out: Path, imgsz: int) -> None: + self._run("coreml", {"pt": str(pt), "out": str(out), "imgsz": imgsz}) + + +@dataclass +class ExportOutcome: + produced: list[str] = field(default_factory=list) + dropped: list[dict[str, str]] = field(default_factory=list) + + def drop(self, export: str, reason: str) -> None: + self.dropped.append({"export": export, "reason": reason[:REASON_CHARS]}) + + +def artefact_names(catalog_id: str) -> dict[str, str]: + return { + "pt": f"{catalog_id}.pt", + "onnx": f"{catalog_id}.onnx", + "openvino-fp16": f"{catalog_id}-fp16.xml", + "openvino-int8": f"{catalog_id}-int8.xml", + "coreml-fp16": f"{catalog_id}.mlpackage", + } + + +def _optional(outcome: ExportOutcome, export: str, run) -> None: + try: + run() + outcome.produced.append(export) + except Exception as exc: # isolated: a format is dropped, the job goes on + outcome.drop(export, str(exc) or type(exc).__name__) + + +def export_all( + exports: list[str], + best_pt: Path, + catalog_id: str, + imgsz: int, + calib: list[Path], + staging: Path, + toolchain: Toolchain, +) -> ExportOutcome: + names = artefact_names(catalog_id) + staging.mkdir(parents=True, exist_ok=True) + shutil.copyfile(best_pt, staging / names["pt"]) + onnx = staging / names["onnx"] + outcome = ExportOutcome() + try: + toolchain.onnx(best_pt, onnx, imgsz) + except Exception as exc: + raise TrainerError("export", "export-onnx", f"ONNX export failed: {str(exc)[:REASON_CHARS]}", EXIT_INTERNAL) from exc + outcome.produced.append("onnx") + if "openvino-fp16" in exports: + _optional(outcome, "openvino-fp16", lambda: toolchain.openvino(onnx, staging / names["openvino-fp16"], True)) + if "openvino-int8" in exports: + if not calib: + outcome.drop("openvino-int8", "no calibration frames in the train split") + else: + _optional( + outcome, + "openvino-int8", + lambda: toolchain.openvino_int8(onnx, staging / names["openvino-int8"], calib, imgsz), + ) + if "coreml-fp16" in exports: + _optional(outcome, "coreml-fp16", lambda: toolchain.coreml(best_pt, staging / names["coreml-fp16"], imgsz)) + return outcome + + +def remove_export(staging: Path, catalog_id: str, export: str) -> None: + """Delete one format's files from the staging dir (the int8 gate).""" + main = staging / artefact_names(catalog_id)[export] + for path in (main, main.with_suffix(".bin")): + if path.is_dir(): + shutil.rmtree(path) + elif path.exists(): + path.unlink() + + +def calibration_frames(train_images: Path, max_frames: int) -> list[Path]: + """The first ``max_frames`` of the user's own TRAIN frames, in name order.""" + return sorted(p for p in train_images.glob("*.jpg"))[:max_frames] diff --git a/camstack_train/floors.py b/camstack_train/floors.py new file mode 100644 index 0000000000000000000000000000000000000000..abfed3714c41ac891a23760124b87509cec115f0 --- /dev/null +++ b/camstack_train/floors.py @@ -0,0 +1,82 @@ +"""The candidate's per-class floors, by the D739 rule, with their reasons. + +The rule itself is the harness's (``_vendor/camstack_eval/floors.py``): the +lowest grid floor at which the candidate raises no more false positives on the +holdout than the baseline does at its shipped floor; a class with fewer than +``minSupport`` holdout positives inherits the baseline's floor; the result is +clamped to ±``clamp`` of it. This module turns the harness's per-class +proposals into the ``floors`` block of result.json, with one human note per +class that did not get a fitted floor — and inherits every floor when there +was no holdout to fit on. +""" +from __future__ import annotations + +from typing import Optional, Sequence + +from ._vendor.camstack_eval.floors import FloorPolicy, ProposedFloor, ScoredDetection, propose_floor +from .spec import FLOOR_CLASSES + + +def propose( + baseline: Sequence[ScoredDetection], + baseline_floor: float, + candidate: Sequence[ScoredDetection], + positives: int, + policy: FloorPolicy, +) -> ProposedFloor: + return propose_floor(baseline, baseline_floor, candidate, positives, policy) + + +def note_for(cls: str, proposal: dict, positives: int, baseline_floor: float, policy: FloorPolicy) -> Optional[str]: + reason = proposal["reason"] + if reason == "low-support": + return f"{cls}: {positives} holdout positives < {policy.min_support}, inherits {baseline_floor}" + if reason == "clamped": + fitted = proposal.get("fitted") + fit = "no grid floor met the false-positive budget" if fitted is None else f"fitted {fitted}" + return f"{cls}: {fit}, clamped to {proposal['value']} (±{policy.clamp} of {baseline_floor})" + return None + + +def floors_from_report( + report: Optional[dict], baseline_floors: dict[str, float], policy: FloorPolicy +) -> tuple[dict[str, float], list[str]]: + """(floors, notes) for one candidate report; no report ⇒ the baseline's floors.""" + if report is None: + return {c: baseline_floors[c] for c in FLOOR_CLASSES}, [ + "no complete holdout frame: every floor inherits the baseline" + ] + floors: dict[str, float] = {} + notes: list[str] = [] + for cls in FLOOR_CLASSES: + entry = report["classes"].get(cls) + if entry is None: + floors[cls] = baseline_floors[cls] + notes.append(f"{cls}: not evaluated, inherits {baseline_floors[cls]}") + continue + proposal = entry["proposedFloor"] + floors[cls] = float(proposal["value"]) + note = note_for(cls, proposal, int(entry["positives"]), baseline_floors[cls], policy) + if note is not None: + notes.append(note) + return floors, notes + + +def floors_block( + candidate: Optional[dict], + candidate_int8: Optional[dict], + baseline_floors: dict[str, float], + policy: FloorPolicy, + int8_present: bool, +) -> dict: + base, notes = floors_from_report(candidate, baseline_floors, policy) + int8 = None + if int8_present: + int8, int8_notes = floors_from_report(candidate_int8, baseline_floors, policy) + notes = notes + [f"int8 {n}" for n in int8_notes] + return { + "baseline": {c: baseline_floors[c] for c in FLOOR_CLASSES}, + "candidate": base, + "candidateInt8": int8, + "notes": notes, + } diff --git a/camstack_train/manifest.py b/camstack_train/manifest.py new file mode 100644 index 0000000000000000000000000000000000000000..ee764bd69aeb22fd0658da3ed2b6988c50fc2a65 --- /dev/null +++ b/camstack_train/manifest.py @@ -0,0 +1,166 @@ +"""``result.json`` (schema ``camstack.training-result/v1``) and ``bundle.tar``. + +``bundle.tar`` holds every artefact under its FLAT catalog name (a Core ML +package as its directory), and ``files[]`` lists one row per regular file in it +with its size and sha256, so the hub verifies what it fetched file by file. + +``catalogDraft`` is a ``ModelCatalogEntry`` per variant whose format URLs are +RELATIVE file names (the hub rewrites them to ``camstack-local://``), with the +revision every format carries under D743/D744: the sha256 of its weight file +(the ONNX itself, an IR's ``.bin``, a Core ML package's ``weight.bin``). +""" +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import os +import platform +import tarfile +from pathlib import Path +from typing import Optional + +from . import __version__ +from .dataset import sha256_file +from .spec import JobSpec + +RESULT_SCHEMA = "camstack.training-result/v1" +LICENCE = "AGPL-3.0 (Ultralytics derivative)" +MLPACKAGE_FILES = ( + "Manifest.json", + "Data/com.apple.CoreML/model.mlmodel", + "Data/com.apple.CoreML/weights/weight.bin", +) +TOOLCHAIN_PACKAGES = ("torch", "ultralytics", "openvino", "nncf", "coremltools", "onnx") + + +def toolchain_versions() -> dict[str, Optional[str]]: + out: dict[str, Optional[str]] = {"python": platform.python_version()} + for name in TOOLCHAIN_PACKAGES: + try: + out[name] = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + out[name] = None + return out + + +def trainer_block() -> dict[str, str]: + return {"version": __version__, "revision": os.environ.get("TRAINER_REVISION", "local")} + + +def classify(rel: str, catalog_id: str) -> tuple[str, str]: + """(format, variant) of a bundle file.""" + if rel.startswith(f"{catalog_id}.mlpackage/"): + return "coreml", "base" + if rel.endswith(".pt"): + return "pt", "base" + if rel.endswith(".onnx"): + return "onnx", "base" + return "openvino", "int8" if rel.startswith(f"{catalog_id}-int8.") else "base" + + +def file_rows(staging: Path, catalog_id: str) -> list[dict]: + rows = [] + for path in sorted(p for p in staging.rglob("*") if p.is_file()): + rel = path.relative_to(staging).as_posix() + fmt, variant = classify(rel, catalog_id) + rows.append({"path": rel, "bytes": path.stat().st_size, "sha256": sha256_file(path), "format": fmt, "variant": variant}) + return rows + + +def write_bundle(staging: Path, rows: list[dict], out: Path) -> dict: + """A plain, deterministic tar of exactly the listed files.""" + out.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(out, "w", format=tarfile.PAX_FORMAT) as tar: + for row in rows: + info = tar.gettarinfo(str(staging / row["path"]), arcname=row["path"]) + info.mtime, info.uid, info.gid, info.uname, info.gname = 0, 0, 0, "", "" + with (staging / row["path"]).open("rb") as handle: + tar.addfile(info, handle) + return {"path": out.name, "bytes": out.stat().st_size, "sha256": sha256_file(out)} + + +def _row(rows: list[dict], path: str) -> Optional[dict]: + return next((r for r in rows if r["path"] == path), None) + + +def _mb(*rows: Optional[dict]) -> float: + return round(sum(r["bytes"] for r in rows if r is not None) / 1e6, 2) + + +def _openvino_format(rows: list[dict], stem: str) -> Optional[dict]: + xml, weights = _row(rows, f"{stem}.xml"), _row(rows, f"{stem}.bin") + if xml is None or weights is None: + return None + return {"url": xml["path"], "files": [weights["path"]], "runtimes": ["python"], "sizeMB": _mb(xml, weights), "revision": weights["sha256"]} + + +def _formats_base(rows: list[dict], cid: str) -> dict: + formats: dict = {} + onnx = _row(rows, f"{cid}.onnx") + if onnx is not None: + formats["onnx"] = {"url": onnx["path"], "sizeMB": _mb(onnx), "revision": onnx["sha256"]} + ov = _openvino_format(rows, f"{cid}-fp16") + if ov is not None: + formats["openvino"] = ov + members = [_row(rows, f"{cid}.mlpackage/{m}") for m in MLPACKAGE_FILES] + if all(m is not None for m in members): + formats["coreml"] = { + "url": f"{cid}.mlpackage", + "isDirectory": True, + "files": list(MLPACKAGE_FILES), + "runtimes": ["python"], + "sizeMB": _mb(*members), + "revision": members[2]["sha256"], + } + return formats + + +def _entry(job: JobSpec, entry_id: str, name: str, description: str, formats: dict, floors: dict) -> dict: + return { + "id": entry_id, + "name": name, + "description": description, + "inputSize": {"width": job.base.imgsz, "height": job.base.imgsz}, + "labels": [], + "preprocessMode": "letterbox", + "defaultClassFloors": floors, + "formats": formats, + } + + +def catalog_draft(job: JobSpec, rows: list[dict], floors: dict) -> dict: + cid = job.catalog.id + origin = f"fine-tuned from {job.base.model_id} on dataset version {job.dataset.version_id}" + base = _entry(job, cid, job.catalog.name, f"{job.catalog.name} — {origin}", _formats_base(rows, cid), floors["candidate"]) + int8 = None + int8_format = _openvino_format(rows, f"{cid}-int8") + if int8_format is not None and floors.get("candidateInt8") is not None: + int8 = _entry( + job, f"{cid}-int8", f"{job.catalog.name} (INT8)", + f"{job.catalog.name} — OpenVINO INT8 calibrated on the user's own frames; {origin}", + {"openvino": int8_format}, floors["candidateInt8"], + ) + return {"base": base, "int8": int8} + + +def result_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write_result(path: Path, result: dict) -> str: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(result, indent=2, sort_keys=False) + "\n", "utf-8") + return result_sha256(path) + + +def failed_result(job_key: str, error: dict) -> dict: + return { + "schema": RESULT_SCHEMA, + "jobKey": job_key, + "status": "failed", + "error": error, + "trainer": trainer_block(), + "toolchain": toolchain_versions(), + "licence": LICENCE, + } diff --git a/camstack_train/replay.py b/camstack_train/replay.py new file mode 100644 index 0000000000000000000000000000000000000000..d04c14296ebbb587f354e03f0d9ec1b213255721 --- /dev/null +++ b/camstack_train/replay.py @@ -0,0 +1,102 @@ +"""COCO replay: public images mixed into a detector fine-tune. + +A fine-tune on a few thousand frames of ~10 cameras forgets what the COCO head +knew about everything those cameras never show. Mixing a fixed slice of COCO +val2017 back in keeps the 80 classes alive. The slice is pinned: the images of +val2017 that hold at least one person / vehicle / animal, in file-name order, +the first ``count`` of them — the same images on every run. + +Nothing here is uploaded by the hub: the container fetches the public files +once into the provider's cache volume (``/cache/coco``) and reuses them. All +80 COCO classes are labelled (the head is kept whole). + +``fetch`` is injectable: tests never touch the network. +""" +from __future__ import annotations + +import io +import json +import urllib.request +import zipfile +from pathlib import Path +from typing import Callable + +from ._vendor.camstack_eval.classes import DETECTOR_CLASSES, macro_of + +ANNOTATIONS_URL = "http://images.cocodataset.org/annotations/annotations_trainval2017.zip" +ANNOTATIONS_MEMBER = "annotations/instances_val2017.json" +IMAGE_URL = "http://images.cocodataset.org/val2017/{file}" + +Fetch = Callable[[str], bytes] + + +def http_fetch(url: str) -> bytes: + with urllib.request.urlopen(url, timeout=120) as response: # noqa: S310 (fixed public URLs) + return response.read() + + +def load_annotations(cache: Path, fetch: Fetch) -> dict: + """``instances_val2017.json``, fetched once into ``cache``.""" + target = cache / "instances_val2017.json" + if not target.is_file(): + cache.mkdir(parents=True, exist_ok=True) + archive = zipfile.ZipFile(io.BytesIO(fetch(ANNOTATIONS_URL))) + target.write_bytes(archive.read(ANNOTATIONS_MEMBER)) + return json.loads(target.read_text("utf-8")) + + +def select_images(annotations: dict, count: int) -> list[dict]: + """The pinned slice: images with a detector-class object, file-name order.""" + names = {c["id"]: c["name"] for c in annotations["categories"]} + wanted: set[int] = set() + for a in annotations["annotations"]: + if not a.get("iscrowd") and macro_of(names[a["category_id"]]) in DETECTOR_CLASSES: + wanted.add(a["image_id"]) + images = sorted((im for im in annotations["images"] if im["id"] in wanted), key=lambda im: im["file_name"]) + return images[:count] + + +def yolo_lines(annotations: dict, image: dict, coco80: list[str]) -> list[str]: + names = {c["id"]: c["name"] for c in annotations["categories"]} + w, h = float(image["width"]), float(image["height"]) + lines = [] + for a in annotations["annotations"]: + if a["image_id"] != image["id"] or a.get("iscrowd"): + continue + name = names[a["category_id"]] + if name not in coco80: + continue + x, y, bw, bh = a["bbox"] + if bw <= 0 or bh <= 0: + continue + lines.append( + f"{coco80.index(name)} {(x + bw / 2) / w:.6f} {(y + bh / 2) / h:.6f} {bw / w:.6f} {bh / h:.6f}" + ) + return lines + + +def prepare_replay(count: int, cache: Path, out_root: Path, coco80: list[str], fetch: Fetch = http_fetch) -> int: + """Write ``count`` replay images + labels under ``out_root/images|labels/replay``.""" + if count <= 0: + return 0 + annotations = load_annotations(cache, fetch) + images_dir = out_root / "images" / "replay" + labels_dir = out_root / "labels" / "replay" + images_dir.mkdir(parents=True, exist_ok=True) + labels_dir.mkdir(parents=True, exist_ok=True) + by_image: dict[int, list[dict]] = {} + for a in annotations["annotations"]: + by_image.setdefault(a["image_id"], []).append(a) + indexed = {**annotations, "annotations": []} + written = 0 + for image in select_images(annotations, count): + cached = cache / "val2017" / image["file_name"] + if not cached.is_file(): + cached.parent.mkdir(parents=True, exist_ok=True) + cached.write_bytes(fetch(IMAGE_URL.format(file=image["file_name"]))) + stem = Path(image["file_name"]).stem + (images_dir / f"{stem}.jpg").write_bytes(cached.read_bytes()) + lines = yolo_lines({**indexed, "annotations": by_image.get(image["id"], [])}, image, coco80) + (labels_dir / f"{stem}.txt").write_text("".join(line + "\n" for line in lines), "utf-8") + written += 1 + return written diff --git a/camstack_train/result_model.py b/camstack_train/result_model.py new file mode 100644 index 0000000000000000000000000000000000000000..2b1ddea8f68df6c775b58613a933fadf33207fd4 --- /dev/null +++ b/camstack_train/result_model.py @@ -0,0 +1,138 @@ +"""The shape of ``result.json`` (``camstack.training-result/v1``), as pydantic. + +``manifest.py`` builds the document as plain dicts; this model is the contract +it is checked against (in the tests, and by ``tools/gen_schemas.py`` into +``schemas/result.schema.json``, which the hub mirrors in zod). The harness's +per-class reports stay open (``dict``): their shape is the harness's own. +""" +from __future__ import annotations + +from typing import Any, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field + + +class _Open(BaseModel): + model_config = ConfigDict(extra="forbid", protected_namespaces=()) + + +class ErrorBlock(_Open): + stage: str + code: str + message: str + + +class TrainerBlock(_Open): + version: str + revision: str + + +class Frames(_Open): + train: int + val: int + holdout: int + + +class DatasetBlock(_Open): + versionId: str + annotationHash: str + sha256: str + frames: Frames + holdoutCameras: list[int] + replayImages: int + + +class TrainingBlock(_Open): + epochsRun: int + bestEpoch: int + bestMap50: float + bestMap50_95: float + seconds: float + + +class EvaluationBlock(_Open): + holdoutFrames: int + baseline: Optional[dict[str, Any]] + candidate: Optional[dict[str, Any]] + candidateInt8: Optional[dict[str, Any]] + recallDeltaByCamera: Optional[dict[str, dict[str, float]]] + map50: dict[str, Optional[float]] + + +class FloorsBlock(_Open): + baseline: dict[str, float] + candidate: dict[str, float] + candidateInt8: Optional[dict[str, float]] + notes: list[str] + + +class Int8Block(_Open): + kept: bool + map50Fp16: Optional[float] + map50Int8: Optional[float] + reason: Optional[str] + + +class Dropped(_Open): + export: str + reason: str + + +class FileRow(_Open): + path: str + bytes: int + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + format: Literal["onnx", "openvino", "coreml", "pt"] + variant: Literal["base", "int8"] + + +class Bundle(_Open): + path: Literal["bundle.tar"] + bytes: int + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class FormatEntry(_Open): + url: str + sizeMB: float + revision: str + files: Optional[list[str]] = None + runtimes: Optional[list[Literal["python"]]] = None + isDirectory: Optional[bool] = None + + +class CatalogEntryDraft(_Open): + id: str + name: str + description: str + inputSize: dict[Literal["width", "height"], int] + labels: list[Any] + preprocessMode: Literal["letterbox"] + defaultClassFloors: dict[str, float] + formats: dict[Literal["onnx", "openvino", "coreml"], FormatEntry] + + +class CatalogDraft(_Open): + base: CatalogEntryDraft + int8: Optional[CatalogEntryDraft] + + +class TrainingResult(_Open): + """A succeeded result carries every block; a failed one the error and whatever was known.""" + + schema_: Literal["camstack.training-result/v1"] = Field(alias="schema") + jobKey: str + status: Literal["succeeded", "failed"] + error: Optional[ErrorBlock] + trainer: TrainerBlock + toolchain: dict[str, Optional[str]] + dataset: Optional[DatasetBlock] = None + training: Optional[TrainingBlock] = None + evaluation: Optional[EvaluationBlock] = None + floors: Optional[FloorsBlock] = None + int8: Optional[Int8Block] = None + exportsDropped: Optional[list[Dropped]] = None + files: Optional[list[FileRow]] = None + bundle: Optional[Bundle] = None + catalogDraft: Optional[CatalogDraft] = None + licence: str diff --git a/camstack_train/run.py b/camstack_train/run.py new file mode 100644 index 0000000000000000000000000000000000000000..e778126ce321f154a564b1a6e7ce55f176abc10b --- /dev/null +++ b/camstack_train/run.py @@ -0,0 +1,167 @@ +"""One training job, end to end: prepare → replay → train → export → eval → package. + +Everything heavy is injected (:class:`Deps`) so the orchestration is tested +without torch, Ultralytics or OpenVINO. ``result.json`` is written on every +exit, success or failure, and the last line is always a ``done`` event. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +from .dataset import coco_80, extract_archive, frames_from_manifest, verify_archive, write_layout +from .errors import EXIT_INTERNAL, EXIT_OK, TrainerError +from .evaluate import DetectorFactory, Evaluation, baseline_model, evaluate, harness_detector, holdout_frames, map50, policy_of, split_reports +from .events import Emitter +from .export import SubprocessToolchain, Toolchain, calibration_frames, export_all, remove_export +from .floors import floors_block +from .manifest import ( + LICENCE, RESULT_SCHEMA, catalog_draft, failed_result, file_rows, toolchain_versions, + trainer_block, write_bundle, write_result, +) +from .replay import Fetch, http_fetch, prepare_replay +from .spec import JobSpec +from .split import SplitResult, assign +from .train_det import Trainer, TrainOutcome, ultralytics_train, write_data_yaml + + +@dataclass +class Deps: + fetch: Fetch = http_fetch + trainer: Trainer = ultralytics_train + toolchain: Toolchain = field(default_factory=SubprocessToolchain) + detector: DetectorFactory = harness_detector + emitter: Emitter = field(default_factory=Emitter) + + +@dataclass(frozen=True) +class Paths: + job: Path + dataset: Path + out: Path + cache: Path + work: Path + + +def _int8_gate(job: JobSpec, evaluation: Evaluation, produced: bool) -> dict: + fp16, int8 = map50(evaluation.candidate), map50(evaluation.candidate_int8) + if not produced: + return {"kept": False, "map50Fp16": fp16, "map50Int8": None, "reason": "not exported"} + if fp16 is None or int8 is None: + return {"kept": True, "map50Fp16": fp16, "map50Int8": int8, "reason": "no holdout to gate on"} + drop = round(fp16 - int8, 4) + if drop > job.int8_gate.max_map50_drop: + return {"kept": False, "map50Fp16": fp16, "map50Int8": int8, "reason": f"mAP50 drop {drop} > {job.int8_gate.max_map50_drop}"} + return {"kept": True, "map50Fp16": fp16, "map50Int8": int8, "reason": None} + + +def _prepare(job: JobSpec, paths: Paths, emitter: Emitter) -> tuple[SplitResult, Path, Path]: + emitter.stage("prepare") + verify_archive(paths.dataset, job.dataset.sha256) + source = paths.work / "export" + manifest = extract_archive(paths.dataset, source) + split = assign(frames_from_manifest(manifest), job.split) + yolo = paths.work / "yolo" + counts = write_layout({"train": list(split.train), "val": list(split.val)}, source, yolo) + emitter.metric("frames.train", counts.frames.get("train", 0)) + emitter.metric("frames.val", counts.frames.get("val", 0)) + emitter.metric("frames.holdout", len(split.holdout)) + return split, source, yolo + + +def _dataset_block(job: JobSpec, split: SplitResult, replayed: int) -> dict: + return { + "versionId": job.dataset.version_id, + "annotationHash": job.dataset.annotation_hash, + "sha256": job.dataset.sha256, + "frames": {"train": len(split.train), "val": len(split.val), "holdout": len(split.holdout)}, + "holdoutCameras": list(split.holdout_cameras), + "replayImages": replayed, + } + + +def _training_block(outcome: TrainOutcome) -> dict: + return { + "epochsRun": outcome.epochs_run, + "bestEpoch": outcome.best_epoch, + "bestMap50": outcome.best_map50, + "bestMap50_95": outcome.best_map50_95, + "seconds": outcome.seconds, + } + + +def execute(job: JobSpec, paths: Paths, deps: Deps, partial: dict) -> dict: + emitter = deps.emitter + split, source, yolo = _prepare(job, paths, emitter) + emitter.stage("replay") + names = coco_80() + replayed = prepare_replay(job.replay.count, paths.cache / "coco", yolo, names, deps.fetch) + partial["dataset"] = _dataset_block(job, split, replayed) + emitter.stage("train") + data_yaml = write_data_yaml(yolo, names, replayed > 0, len(split.val) > 0) + outcome = deps.trainer(job, data_yaml, paths.work, paths.cache, emitter) + partial["training"] = _training_block(outcome) + emitter.stage("export") + staging = paths.work / "artifacts" + calib = calibration_frames(yolo / "images" / "train", job.calibration.max_frames) + exported = export_all(list(job.exports), outcome.best_pt, job.catalog.id, job.base.imgsz, calib, staging, deps.toolchain) + emitter.stage("eval") + int8_path = staging / f"{job.catalog.id}-int8.xml" if "openvino-int8" in exported.produced else None + frames = holdout_frames(source, {f.frame_id for f in split.holdout_eval}) + baseline = baseline_model(job, paths.cache, deps.fetch) if frames else None + evaluation = ( + evaluate(job, frames, baseline, staging / f"{job.catalog.id}.onnx", int8_path, deps.detector) + if baseline is not None else Evaluation(None, None, 0) + ) + gate = _int8_gate(job, evaluation, int8_path is not None) + if int8_path is not None and not gate["kept"]: + remove_export(staging, job.catalog.id, "openvino-int8") + exported.drop("openvino-int8", gate["reason"]) + floors = floors_block(evaluation.candidate, evaluation.candidate_int8, dict(job.baseline.floors), policy_of(job), gate["kept"] and int8_path is not None) + emitter.stage("package") + rows = file_rows(staging, job.catalog.id) + for row in rows: + emitter.artifact(row["path"], row["bytes"], row["sha256"]) + bundle = write_bundle(staging, rows, paths.out / "bundle.tar") + return { + "schema": RESULT_SCHEMA, + "jobKey": job.job_key, + "status": "succeeded", + "error": None, + "trainer": trainer_block(), + "toolchain": toolchain_versions(), + **partial, + "evaluation": split_reports(evaluation), + "floors": floors, + "int8": gate, + "exportsDropped": exported.dropped, + "files": rows, + "bundle": bundle, + "catalogDraft": catalog_draft(job, rows, floors), + "licence": LICENCE, + } + + +def scrub(message: str, paths: Paths) -> str: + """No frame path leaves the container: every one lives under the work dir.""" + return message.replace(str(paths.work), "").replace(str(paths.dataset), "")[:300] + + +def run_job(job: JobSpec, paths: Paths, deps: Optional[Deps] = None) -> int: + deps = deps or Deps() + partial: dict = {} + try: + result = execute(job, paths, deps, partial) + code, status = EXIT_OK, "succeeded" + except TrainerError as exc: + deps.emitter.error(exc.stage, exc.code, exc.message) + result, code, status = {**failed_result(job.job_key, exc.as_dict()), **partial}, exc.exit_code, "failed" + except Exception as exc: # anything unforeseen: still a result and a done line + message = scrub(f"{type(exc).__name__}: {exc}", paths) + error = {"stage": "internal", "code": "internal", "message": message} + deps.emitter.error("internal", "internal", message) + result, code, status = {**failed_result(job.job_key, error), **partial}, EXIT_INTERNAL, "failed" + digest = write_result(paths.out / "result.json", result) + deps.emitter.done(status, digest) + return code diff --git a/camstack_train/spec.py b/camstack_train/spec.py new file mode 100644 index 0000000000000000000000000000000000000000..e80e2a239d17c1a1be00e4df1867ada54fe5cc58 --- /dev/null +++ b/camstack_train/spec.py @@ -0,0 +1,173 @@ +"""``job.json``, schema ``camstack.training/v1``. + +The hub fills EVERY field: nothing here relies on a default, and an unknown +key is refused, so a hub and a trainer that disagree about the contract fail +loudly at the first line instead of training on a silently ignored setting. +An unknown ``schema`` exits 64 before anything is read. +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator +from pydantic.alias_generators import to_camel + +from .errors import TrainerError, spec_error + +JOB_SCHEMA = "camstack.training/v1" + +ExportName = Literal["onnx", "openvino-fp16", "openvino-int8", "coreml-fp16"] + +#: Macro classes a root detector's floors are kept for. +FLOOR_CLASSES = ("person", "vehicle", "animal") + +#: A catalog id becomes file names on every node: lower-case, no path parts. +CATALOG_ID_PATTERN = r"^[a-z0-9][a-z0-9._-]{0,95}$" + + +class _Model(BaseModel): + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + extra="forbid", + frozen=True, + protected_namespaces=(), + ) + + +class Target(_Model): + kind: Literal["detector"] + step: Literal["object-detection"] + + +class Base(_Model): + model_id: str = Field(min_length=1) + weights: str = Field(pattern=r"^[A-Za-z0-9._-]+\.pt$") + imgsz: int = Field(ge=64, le=2048) + + +class Baseline(_Model): + model_id: str = Field(min_length=1) + onnx_url: str = Field(pattern=r"^https://") + imgsz: int = Field(ge=64, le=2048) + floors: dict[str, float] + + @field_validator("floors") + @classmethod + def _every_class(cls, value: dict[str, float]) -> dict[str, float]: + missing = [c for c in FLOOR_CLASSES if c not in value] + if missing: + raise ValueError(f"baseline floors miss {', '.join(missing)}") + if any(not 0.0 <= v <= 1.0 for v in value.values()): + raise ValueError("a baseline floor is outside [0, 1]") + return value + + +class Dataset(_Model): + version_id: str = Field(min_length=1) + annotation_hash: str + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class Split(_Model): + holdout_cameras: list[int] + holdout_last_days: int = Field(ge=0, le=365) + val_fraction: float = Field(ge=0.0, le=0.5) + seed: int + + +class Hyper(_Model): + epochs: int = Field(ge=1, le=1000) + batch: int = Field(ge=1, le=512) + lr0: float = Field(gt=0.0, le=1.0) + freeze: int = Field(ge=0, le=100) + patience: int = Field(ge=0, le=1000) + seed: int + + +class Replay(_Model): + source: Literal["coco2017"] + count: int = Field(ge=0, le=20000) + + +class Calibration(_Model): + max_frames: int = Field(ge=1, le=5000) + + +class Floors(_Model): + min_support: int = Field(ge=0) + clamp: float = Field(ge=0.0, le=1.0) + + +class Int8Gate(_Model): + max_map50_drop: float = Field(ge=0.0, le=1.0) + + +class Catalog(_Model): + id: str = Field(pattern=CATALOG_ID_PATTERN) + name: str = Field(min_length=1, max_length=120) + + +class JobSpec(_Model): + schema_: Literal["camstack.training/v1"] = Field(alias="schema") + job_key: str = Field(min_length=1, max_length=64, pattern=r"^[A-Za-z0-9-]+$") + target: Target + base: Base + baseline: Baseline + dataset: Dataset + split: Split + hyper: Hyper + replay: Replay + exports: list[ExportName] = Field(min_length=1) + calibration: Calibration + floors: Floors + int8_gate: Int8Gate + catalog: Catalog + + @field_validator("exports") + @classmethod + def _onnx_first(cls, value: list[str]) -> list[str]: + if "onnx" not in value: + raise ValueError("exports must include onnx: every other format is built from it") + if len(set(value)) != len(value): + raise ValueError("exports lists a format twice") + return value + + +def parse_job(raw: object) -> JobSpec: + """Validate a decoded ``job.json``; any refusal is a spec error (exit 64).""" + if not isinstance(raw, dict): + raise spec_error("bad-spec", "job.json is not an object") + schema = raw.get("schema") + if schema != JOB_SCHEMA: + raise spec_error("unknown-schema", f"job schema {schema!r} is not {JOB_SCHEMA}") + try: + return JobSpec.model_validate(raw) + except ValidationError as exc: + fields = sorted({".".join(str(p) for p in e["loc"]) for e in exc.errors()}) + raise spec_error("bad-spec", f"job.json refused at: {', '.join(fields)}") from exc + + +def load_job(path: Path) -> JobSpec: + try: + raw = json.loads(path.read_text("utf-8")) + except FileNotFoundError as exc: + raise spec_error("bad-spec", "job.json not found") from exc + except json.JSONDecodeError as exc: + raise spec_error("bad-spec", f"job.json is not JSON (line {exc.lineno})") from exc + return parse_job(raw) + + +def job_key_of(path: Path) -> str: + """The ``jobKey`` of a job file even when the rest is refused (for result.json).""" + try: + raw = json.loads(path.read_text("utf-8")) + except (OSError, json.JSONDecodeError): + return "" + key = raw.get("jobKey") if isinstance(raw, dict) else None + return key if isinstance(key, str) else "" + + +__all__ = ["JobSpec", "JOB_SCHEMA", "TrainerError", "parse_job", "load_job", "job_key_of"] diff --git a/camstack_train/split.py b/camstack_train/split.py new file mode 100644 index 0000000000000000000000000000000000000000..105770cfecd967b8377899b3891e449960488387 --- /dev/null +++ b/camstack_train/split.py @@ -0,0 +1,101 @@ +"""The train / val / holdout assignment, deterministic from ``job.split``. + +Never random per frame: two frames a second apart on one camera are near +copies, and splitting them between train and holdout measures memory, not the +model. So: + +- **holdout**: whole cameras (``holdoutCameras``) when the hub named any; + otherwise the last ``holdoutLastDays`` capture days of EVERY camera (a frame + with no capture time is never held out); otherwise nothing. +- **val**: whole (camera, day) groups of the remaining trainable frames, in a + seeded hash order, until ``valFraction`` of them is reached — never the last + train group. +- **train**: everything else that is trainable (``dataset.Frame.trainable``). + +A holdout frame never trains, whether or not it is complete; only complete +holdout frames are evaluated. A shortage of holdout positives does NOT refuse +the job — the floor rule then inherits the baseline's floor. An empty train +split does (exit 65). +""" +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import Optional, Sequence + +from .dataset import Frame +from .errors import dataset_error +from .spec import Split + +DAY_MS = 86_400_000 + + +@dataclass(frozen=True) +class SplitResult: + train: tuple[Frame, ...] + val: tuple[Frame, ...] + holdout: tuple[Frame, ...] + holdout_cameras: tuple[int, ...] + + @property + def holdout_eval(self) -> tuple[Frame, ...]: + return tuple(f for f in self.holdout if f.complete) + + +def day_of(frame: Frame) -> Optional[int]: + return None if frame.captured_at is None else frame.captured_at // DAY_MS + + +def _holdout_by_days(frames: Sequence[Frame], days: int) -> set[str]: + last_day: dict[int, int] = {} + for f in frames: + d = day_of(f) + if d is not None: + last_day[f.device_id] = max(last_day.get(f.device_id, d), d) + held = set() + for f in frames: + d = day_of(f) + if d is not None and d > last_day[f.device_id] - days: + held.add(f.frame_id) + return held + + +def _group_order(seed: int, device_id: int, day: Optional[int]) -> str: + return hashlib.sha256(f"{seed}:{device_id}:{day}".encode()).hexdigest() + + +def _val_groups(frames: Sequence[Frame], fraction: float, seed: int) -> set[tuple[int, Optional[int]]]: + groups: dict[tuple[int, Optional[int]], int] = {} + for f in frames: + key = (f.device_id, day_of(f)) + groups[key] = groups.get(key, 0) + 1 + target = round(fraction * len(frames)) + chosen: set[tuple[int, Optional[int]]] = set() + taken = 0 + ordered = sorted(groups, key=lambda k: _group_order(seed, k[0], k[1])) + for key in ordered: + if taken >= target or len(chosen) >= len(groups) - 1: + break + chosen.add(key) + taken += groups[key] + return chosen + + +def assign(frames: Sequence[Frame], policy: Split) -> SplitResult: + if policy.holdout_cameras: + wanted = set(policy.holdout_cameras) + held = {f.frame_id for f in frames if f.device_id in wanted} + cameras = tuple(sorted(wanted)) + elif policy.holdout_last_days > 0: + held = _holdout_by_days(frames, policy.holdout_last_days) + cameras = () + else: + held, cameras = set(), () + holdout = tuple(f for f in frames if f.frame_id in held) + rest = [f for f in frames if f.frame_id not in held and f.trainable] + val_keys = _val_groups(rest, policy.val_fraction, policy.seed) + val = tuple(f for f in rest if (f.device_id, day_of(f)) in val_keys) + train = tuple(f for f in rest if (f.device_id, day_of(f)) not in val_keys) + if not train: + raise dataset_error("no-train-frames", "no trainable frame is left outside the holdout") + return SplitResult(train=train, val=val, holdout=holdout, holdout_cameras=cameras) diff --git a/camstack_train/train_det.py b/camstack_train/train_det.py new file mode 100644 index 0000000000000000000000000000000000000000..64fb1d6260948a1316504af10f519657071f2615 --- /dev/null +++ b/camstack_train/train_det.py @@ -0,0 +1,131 @@ +"""Fine-tune the YOLOv9 root detector with Ultralytics. + +This is the ONLY module that imports Ultralytics (AGPL-3.0), and only inside +:func:`ultralytics_train`, so the rest of the package — and its tests — never +load it. The hub never imports this package at all. +""" +from __future__ import annotations + +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Optional, Protocol + +from .events import Emitter +from .spec import JobSpec + +#: Where Ultralytics' base weights are cached between jobs (the cache volume). +WEIGHTS_DIR = "weights" + + +@dataclass(frozen=True) +class TrainOutcome: + best_pt: Path + epochs_run: int + best_epoch: int + best_map50: float + best_map50_95: float + seconds: float + + +class Trainer(Protocol): + def __call__(self, job: JobSpec, data_yaml: Path, work: Path, cache: Path, emitter: Emitter) -> TrainOutcome: ... + + +def write_data_yaml(root: Path, names: list[str], with_replay: bool, has_val: bool) -> Path: + """The Ultralytics dataset file. Without a val split, val = train (stated).""" + train = ["images/train"] + (["images/replay"] if with_replay else []) + val = "images/val" if has_val else "images/train" + lines = [f"path: {root.as_posix()}", "train:"] + lines += [f" - {t}" for t in train] + lines.append(f"val: {val}") + lines.append("names:") + lines += [f" {i}: {name!r}" for i, name in enumerate(names)] + path = root / "data.yaml" + path.write_text("\n".join(lines) + "\n", "utf-8") + return path + + +@dataclass +class EpochTracker: + """Turns Ultralytics' per-epoch callback into ``epoch`` events + the best epoch.""" + + emitter: Emitter + epochs: int + started: float + clock: Callable[[], float] = time.monotonic + epochs_run: int = 0 + best_epoch: int = 0 + best_map50: float = 0.0 + best_map50_95: float = 0.0 + + def on_epoch(self, epoch: int, box_loss: Optional[float], metrics: dict[str, Any]) -> None: + map50 = _number(metrics.get("metrics/mAP50(B)")) + map50_95 = _number(metrics.get("metrics/mAP50-95(B)")) + self.epochs_run = epoch + if map50_95 is not None and map50_95 >= self.best_map50_95: + self.best_epoch, self.best_map50_95 = epoch, map50_95 + self.best_map50 = map50 or 0.0 + elapsed = self.clock() - self.started + eta = elapsed / epoch * (self.epochs - epoch) if epoch > 0 else None + self.emitter.epoch(epoch, self.epochs, box_loss, map50, map50_95, None if eta is None else round(eta, 1)) + + +def _number(value: Any) -> Optional[float]: + try: + return None if value is None else round(float(value), 5) + except (TypeError, ValueError): + return None + + +def base_weights(job: JobSpec, cache: Path) -> Path: + """The base checkpoint, downloaded once into the cache volume.""" + from ultralytics.utils.downloads import attempt_download_asset + + target = cache / WEIGHTS_DIR / job.base.weights + target.parent.mkdir(parents=True, exist_ok=True) + if not target.is_file(): + attempt_download_asset(str(target)) + return target + + +def ultralytics_train(job: JobSpec, data_yaml: Path, work: Path, cache: Path, emitter: Emitter) -> TrainOutcome: + import torch + from ultralytics import YOLO + + model = YOLO(str(base_weights(job, cache))) + tracker = EpochTracker(emitter, job.hyper.epochs, time.monotonic()) + + def on_fit_epoch_end(trainer: Any) -> None: + loss = getattr(trainer, "tloss", None) + box_loss = _number(loss[0]) if loss is not None and len(loss) > 0 else None + tracker.on_epoch(int(trainer.epoch) + 1, box_loss, dict(getattr(trainer, "metrics", {}) or {})) + + model.add_callback("on_fit_epoch_end", on_fit_epoch_end) + started = time.monotonic() + model.train( + data=str(data_yaml), + epochs=job.hyper.epochs, + batch=job.hyper.batch, + lr0=job.hyper.lr0, + freeze=job.hyper.freeze or None, + patience=job.hyper.patience, + seed=job.hyper.seed, + deterministic=True, + imgsz=job.base.imgsz, + device=0 if torch.cuda.is_available() else "cpu", + project=str(work / "runs"), + name="train", + exist_ok=True, + plots=False, + verbose=False, + ) + best = work / "runs" / "train" / "weights" / "best.pt" + return TrainOutcome( + best_pt=best, + epochs_run=tracker.epochs_run, + best_epoch=tracker.best_epoch, + best_map50=tracker.best_map50, + best_map50_95=tracker.best_map50_95, + seconds=round(time.monotonic() - started, 1), + ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..dab34a280355ea3d0af38faa73048d3a26d4c4cf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,50 @@ +# Tracked in camstack/server as hf/camstack-trainer/; edit it there, not on the Hub. +[build-system] +requires = ["setuptools>=75", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "camstack-train" +version = "0.1.0" +description = "CamStack detector trainer: fine-tunes the YOLOv9 root on a hub's annotated export" +readme = "README.md" +license = "AGPL-3.0-only" +requires-python = ">=3.10" +# The light core: spec, dataset conversion, split, floors, manifest and the +# vendored evaluation harness. Enough to run the unit tests. +dependencies = [ + "numpy==2.5.1", + "pillow==12.3.0", + "pydantic==2.13.4", + "onnxruntime==1.27.0", +] + +[project.optional-dependencies] +# torch comes from the base image (pytorch/pytorch:2.13.0-cuda12.6-cudnn9-runtime); +# it is deliberately NOT pinned here so pip never replaces the CUDA build. +train = [ + "ultralytics==8.4.98", +] +export = [ + "onnx==1.22.0", + "onnxslim==0.1.94", + "openvino==2026.2.1", + "nncf==3.2.0", + "coremltools==9.0", +] +test = [ + "pytest==9.1.1", +] + +[project.scripts] +camstack-train = "camstack_train.cli:main" + +[tool.setuptools.packages.find] +include = ["camstack_train*"] +namespaces = true + +[tool.setuptools.package-data] +"*" = ["*.py"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/schemas/job.schema.json b/schemas/job.schema.json new file mode 100644 index 0000000000000000000000000000000000000000..871c41203b7f343ca7bb4e9b78cef5e9010faba0 --- /dev/null +++ b/schemas/job.schema.json @@ -0,0 +1,380 @@ +{ + "$defs": { + "Base": { + "additionalProperties": false, + "properties": { + "imgsz": { + "maximum": 2048, + "minimum": 64, + "title": "Imgsz", + "type": "integer" + }, + "modelId": { + "minLength": 1, + "title": "Modelid", + "type": "string" + }, + "weights": { + "pattern": "^[A-Za-z0-9._-]+\\.pt$", + "title": "Weights", + "type": "string" + } + }, + "required": [ + "modelId", + "weights", + "imgsz" + ], + "title": "Base", + "type": "object" + }, + "Baseline": { + "additionalProperties": false, + "properties": { + "floors": { + "additionalProperties": { + "type": "number" + }, + "title": "Floors", + "type": "object" + }, + "imgsz": { + "maximum": 2048, + "minimum": 64, + "title": "Imgsz", + "type": "integer" + }, + "modelId": { + "minLength": 1, + "title": "Modelid", + "type": "string" + }, + "onnxUrl": { + "pattern": "^https://", + "title": "Onnxurl", + "type": "string" + } + }, + "required": [ + "modelId", + "onnxUrl", + "imgsz", + "floors" + ], + "title": "Baseline", + "type": "object" + }, + "Calibration": { + "additionalProperties": false, + "properties": { + "maxFrames": { + "maximum": 5000, + "minimum": 1, + "title": "Maxframes", + "type": "integer" + } + }, + "required": [ + "maxFrames" + ], + "title": "Calibration", + "type": "object" + }, + "Catalog": { + "additionalProperties": false, + "properties": { + "id": { + "pattern": "^[a-z0-9][a-z0-9._-]{0,95}$", + "title": "Id", + "type": "string" + }, + "name": { + "maxLength": 120, + "minLength": 1, + "title": "Name", + "type": "string" + } + }, + "required": [ + "id", + "name" + ], + "title": "Catalog", + "type": "object" + }, + "Dataset": { + "additionalProperties": false, + "properties": { + "annotationHash": { + "title": "Annotationhash", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + }, + "versionId": { + "minLength": 1, + "title": "Versionid", + "type": "string" + } + }, + "required": [ + "versionId", + "annotationHash", + "sha256" + ], + "title": "Dataset", + "type": "object" + }, + "Floors": { + "additionalProperties": false, + "properties": { + "clamp": { + "maximum": 1.0, + "minimum": 0.0, + "title": "Clamp", + "type": "number" + }, + "minSupport": { + "minimum": 0, + "title": "Minsupport", + "type": "integer" + } + }, + "required": [ + "minSupport", + "clamp" + ], + "title": "Floors", + "type": "object" + }, + "Hyper": { + "additionalProperties": false, + "properties": { + "batch": { + "maximum": 512, + "minimum": 1, + "title": "Batch", + "type": "integer" + }, + "epochs": { + "maximum": 1000, + "minimum": 1, + "title": "Epochs", + "type": "integer" + }, + "freeze": { + "maximum": 100, + "minimum": 0, + "title": "Freeze", + "type": "integer" + }, + "lr0": { + "exclusiveMinimum": 0.0, + "maximum": 1.0, + "title": "Lr0", + "type": "number" + }, + "patience": { + "maximum": 1000, + "minimum": 0, + "title": "Patience", + "type": "integer" + }, + "seed": { + "title": "Seed", + "type": "integer" + } + }, + "required": [ + "epochs", + "batch", + "lr0", + "freeze", + "patience", + "seed" + ], + "title": "Hyper", + "type": "object" + }, + "Int8Gate": { + "additionalProperties": false, + "properties": { + "maxMap50Drop": { + "maximum": 1.0, + "minimum": 0.0, + "title": "Maxmap50Drop", + "type": "number" + } + }, + "required": [ + "maxMap50Drop" + ], + "title": "Int8Gate", + "type": "object" + }, + "Replay": { + "additionalProperties": false, + "properties": { + "count": { + "maximum": 20000, + "minimum": 0, + "title": "Count", + "type": "integer" + }, + "source": { + "const": "coco2017", + "title": "Source", + "type": "string" + } + }, + "required": [ + "source", + "count" + ], + "title": "Replay", + "type": "object" + }, + "Split": { + "additionalProperties": false, + "properties": { + "holdoutCameras": { + "items": { + "type": "integer" + }, + "title": "Holdoutcameras", + "type": "array" + }, + "holdoutLastDays": { + "maximum": 365, + "minimum": 0, + "title": "Holdoutlastdays", + "type": "integer" + }, + "seed": { + "title": "Seed", + "type": "integer" + }, + "valFraction": { + "maximum": 0.5, + "minimum": 0.0, + "title": "Valfraction", + "type": "number" + } + }, + "required": [ + "holdoutCameras", + "holdoutLastDays", + "valFraction", + "seed" + ], + "title": "Split", + "type": "object" + }, + "Target": { + "additionalProperties": false, + "properties": { + "kind": { + "const": "detector", + "title": "Kind", + "type": "string" + }, + "step": { + "const": "object-detection", + "title": "Step", + "type": "string" + } + }, + "required": [ + "kind", + "step" + ], + "title": "Target", + "type": "object" + } + }, + "$id": "camstack.training/v1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "base": { + "$ref": "#/$defs/Base" + }, + "baseline": { + "$ref": "#/$defs/Baseline" + }, + "calibration": { + "$ref": "#/$defs/Calibration" + }, + "catalog": { + "$ref": "#/$defs/Catalog" + }, + "dataset": { + "$ref": "#/$defs/Dataset" + }, + "exports": { + "items": { + "enum": [ + "onnx", + "openvino-fp16", + "openvino-int8", + "coreml-fp16" + ], + "type": "string" + }, + "minItems": 1, + "title": "Exports", + "type": "array" + }, + "floors": { + "$ref": "#/$defs/Floors" + }, + "hyper": { + "$ref": "#/$defs/Hyper" + }, + "int8Gate": { + "$ref": "#/$defs/Int8Gate" + }, + "jobKey": { + "maxLength": 64, + "minLength": 1, + "pattern": "^[A-Za-z0-9-]+$", + "title": "Jobkey", + "type": "string" + }, + "replay": { + "$ref": "#/$defs/Replay" + }, + "schema": { + "const": "camstack.training/v1", + "title": "Schema", + "type": "string" + }, + "split": { + "$ref": "#/$defs/Split" + }, + "target": { + "$ref": "#/$defs/Target" + } + }, + "required": [ + "schema", + "jobKey", + "target", + "base", + "baseline", + "dataset", + "split", + "hyper", + "replay", + "exports", + "calibration", + "floors", + "int8Gate", + "catalog" + ], + "title": "JobSpec", + "type": "object" +} diff --git a/schemas/result.schema.json b/schemas/result.schema.json new file mode 100644 index 0000000000000000000000000000000000000000..fc87a39d3a92982eb6c629ee147ef1f9596daafe --- /dev/null +++ b/schemas/result.schema.json @@ -0,0 +1,756 @@ +{ + "$defs": { + "Bundle": { + "additionalProperties": false, + "properties": { + "bytes": { + "title": "Bytes", + "type": "integer" + }, + "path": { + "const": "bundle.tar", + "title": "Path", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "path", + "bytes", + "sha256" + ], + "title": "Bundle", + "type": "object" + }, + "CatalogDraft": { + "additionalProperties": false, + "properties": { + "base": { + "$ref": "#/$defs/CatalogEntryDraft" + }, + "int8": { + "anyOf": [ + { + "$ref": "#/$defs/CatalogEntryDraft" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "base", + "int8" + ], + "title": "CatalogDraft", + "type": "object" + }, + "CatalogEntryDraft": { + "additionalProperties": false, + "properties": { + "defaultClassFloors": { + "additionalProperties": { + "type": "number" + }, + "title": "Defaultclassfloors", + "type": "object" + }, + "description": { + "title": "Description", + "type": "string" + }, + "formats": { + "additionalProperties": { + "$ref": "#/$defs/FormatEntry" + }, + "propertyNames": { + "enum": [ + "onnx", + "openvino", + "coreml" + ] + }, + "title": "Formats", + "type": "object" + }, + "id": { + "title": "Id", + "type": "string" + }, + "inputSize": { + "additionalProperties": { + "type": "integer" + }, + "propertyNames": { + "enum": [ + "width", + "height" + ] + }, + "title": "Inputsize", + "type": "object" + }, + "labels": { + "items": {}, + "title": "Labels", + "type": "array" + }, + "name": { + "title": "Name", + "type": "string" + }, + "preprocessMode": { + "const": "letterbox", + "title": "Preprocessmode", + "type": "string" + } + }, + "required": [ + "id", + "name", + "description", + "inputSize", + "labels", + "preprocessMode", + "defaultClassFloors", + "formats" + ], + "title": "CatalogEntryDraft", + "type": "object" + }, + "DatasetBlock": { + "additionalProperties": false, + "properties": { + "annotationHash": { + "title": "Annotationhash", + "type": "string" + }, + "frames": { + "$ref": "#/$defs/Frames" + }, + "holdoutCameras": { + "items": { + "type": "integer" + }, + "title": "Holdoutcameras", + "type": "array" + }, + "replayImages": { + "title": "Replayimages", + "type": "integer" + }, + "sha256": { + "title": "Sha256", + "type": "string" + }, + "versionId": { + "title": "Versionid", + "type": "string" + } + }, + "required": [ + "versionId", + "annotationHash", + "sha256", + "frames", + "holdoutCameras", + "replayImages" + ], + "title": "DatasetBlock", + "type": "object" + }, + "Dropped": { + "additionalProperties": false, + "properties": { + "export": { + "title": "Export", + "type": "string" + }, + "reason": { + "title": "Reason", + "type": "string" + } + }, + "required": [ + "export", + "reason" + ], + "title": "Dropped", + "type": "object" + }, + "ErrorBlock": { + "additionalProperties": false, + "properties": { + "code": { + "title": "Code", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "stage": { + "title": "Stage", + "type": "string" + } + }, + "required": [ + "stage", + "code", + "message" + ], + "title": "ErrorBlock", + "type": "object" + }, + "EvaluationBlock": { + "additionalProperties": false, + "properties": { + "baseline": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Baseline" + }, + "candidate": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Candidate" + }, + "candidateInt8": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Candidateint8" + }, + "holdoutFrames": { + "title": "Holdoutframes", + "type": "integer" + }, + "map50": { + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "title": "Map50", + "type": "object" + }, + "recallDeltaByCamera": { + "anyOf": [ + { + "additionalProperties": { + "additionalProperties": { + "type": "number" + }, + "type": "object" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Recalldeltabycamera" + } + }, + "required": [ + "holdoutFrames", + "baseline", + "candidate", + "candidateInt8", + "recallDeltaByCamera", + "map50" + ], + "title": "EvaluationBlock", + "type": "object" + }, + "FileRow": { + "additionalProperties": false, + "properties": { + "bytes": { + "title": "Bytes", + "type": "integer" + }, + "format": { + "enum": [ + "onnx", + "openvino", + "coreml", + "pt" + ], + "title": "Format", + "type": "string" + }, + "path": { + "title": "Path", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + }, + "variant": { + "enum": [ + "base", + "int8" + ], + "title": "Variant", + "type": "string" + } + }, + "required": [ + "path", + "bytes", + "sha256", + "format", + "variant" + ], + "title": "FileRow", + "type": "object" + }, + "FloorsBlock": { + "additionalProperties": false, + "properties": { + "baseline": { + "additionalProperties": { + "type": "number" + }, + "title": "Baseline", + "type": "object" + }, + "candidate": { + "additionalProperties": { + "type": "number" + }, + "title": "Candidate", + "type": "object" + }, + "candidateInt8": { + "anyOf": [ + { + "additionalProperties": { + "type": "number" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Candidateint8" + }, + "notes": { + "items": { + "type": "string" + }, + "title": "Notes", + "type": "array" + } + }, + "required": [ + "baseline", + "candidate", + "candidateInt8", + "notes" + ], + "title": "FloorsBlock", + "type": "object" + }, + "FormatEntry": { + "additionalProperties": false, + "properties": { + "files": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Files" + }, + "isDirectory": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Isdirectory" + }, + "revision": { + "title": "Revision", + "type": "string" + }, + "runtimes": { + "anyOf": [ + { + "items": { + "const": "python", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Runtimes" + }, + "sizeMB": { + "title": "Sizemb", + "type": "number" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "url", + "sizeMB", + "revision" + ], + "title": "FormatEntry", + "type": "object" + }, + "Frames": { + "additionalProperties": false, + "properties": { + "holdout": { + "title": "Holdout", + "type": "integer" + }, + "train": { + "title": "Train", + "type": "integer" + }, + "val": { + "title": "Val", + "type": "integer" + } + }, + "required": [ + "train", + "val", + "holdout" + ], + "title": "Frames", + "type": "object" + }, + "Int8Block": { + "additionalProperties": false, + "properties": { + "kept": { + "title": "Kept", + "type": "boolean" + }, + "map50Fp16": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Map50Fp16" + }, + "map50Int8": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Map50Int8" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reason" + } + }, + "required": [ + "kept", + "map50Fp16", + "map50Int8", + "reason" + ], + "title": "Int8Block", + "type": "object" + }, + "TrainerBlock": { + "additionalProperties": false, + "properties": { + "revision": { + "title": "Revision", + "type": "string" + }, + "version": { + "title": "Version", + "type": "string" + } + }, + "required": [ + "version", + "revision" + ], + "title": "TrainerBlock", + "type": "object" + }, + "TrainingBlock": { + "additionalProperties": false, + "properties": { + "bestEpoch": { + "title": "Bestepoch", + "type": "integer" + }, + "bestMap50": { + "title": "Bestmap50", + "type": "number" + }, + "bestMap50_95": { + "title": "Bestmap50 95", + "type": "number" + }, + "epochsRun": { + "title": "Epochsrun", + "type": "integer" + }, + "seconds": { + "title": "Seconds", + "type": "number" + } + }, + "required": [ + "epochsRun", + "bestEpoch", + "bestMap50", + "bestMap50_95", + "seconds" + ], + "title": "TrainingBlock", + "type": "object" + } + }, + "$id": "camstack.training-result/v1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "A succeeded result carries every block; a failed one the error and whatever was known.", + "properties": { + "bundle": { + "anyOf": [ + { + "$ref": "#/$defs/Bundle" + }, + { + "type": "null" + } + ], + "default": null + }, + "catalogDraft": { + "anyOf": [ + { + "$ref": "#/$defs/CatalogDraft" + }, + { + "type": "null" + } + ], + "default": null + }, + "dataset": { + "anyOf": [ + { + "$ref": "#/$defs/DatasetBlock" + }, + { + "type": "null" + } + ], + "default": null + }, + "error": { + "anyOf": [ + { + "$ref": "#/$defs/ErrorBlock" + }, + { + "type": "null" + } + ] + }, + "evaluation": { + "anyOf": [ + { + "$ref": "#/$defs/EvaluationBlock" + }, + { + "type": "null" + } + ], + "default": null + }, + "exportsDropped": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Dropped" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Exportsdropped" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/FileRow" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Files" + }, + "floors": { + "anyOf": [ + { + "$ref": "#/$defs/FloorsBlock" + }, + { + "type": "null" + } + ], + "default": null + }, + "int8": { + "anyOf": [ + { + "$ref": "#/$defs/Int8Block" + }, + { + "type": "null" + } + ], + "default": null + }, + "jobKey": { + "title": "Jobkey", + "type": "string" + }, + "licence": { + "title": "Licence", + "type": "string" + }, + "schema": { + "const": "camstack.training-result/v1", + "title": "Schema", + "type": "string" + }, + "status": { + "enum": [ + "succeeded", + "failed" + ], + "title": "Status", + "type": "string" + }, + "toolchain": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "title": "Toolchain", + "type": "object" + }, + "trainer": { + "$ref": "#/$defs/TrainerBlock" + }, + "training": { + "anyOf": [ + { + "$ref": "#/$defs/TrainingBlock" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "schema", + "jobKey", + "status", + "error", + "trainer", + "toolchain", + "licence" + ], + "title": "TrainingResult", + "type": "object" +} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..a885844540f4f2a508cea1eb0a9b9758aa7e1715 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,97 @@ +"""Fixtures: a synthetic hub export in the real manifest format, and a job.""" +from __future__ import annotations + +import copy +import hashlib +import io +import json +import tarfile +from pathlib import Path +from typing import Callable + +import pytest +from PIL import Image + +DAY = 86_400_000 +T0 = 1_760_000_000_000 # a capture instant (epoch ms) + +JOB = { + "schema": "camstack.training/v1", + "jobKey": "0b6c1f7e-6a4e-4a8e-9f0b-3c2a1d4e5f60", + "target": {"kind": "detector", "step": "object-detection"}, + "base": {"modelId": "yolov9m-640", "weights": "yolov9m.pt", "imgsz": 640}, + "baseline": { + "modelId": "yolov9m-640", + "onnxUrl": "https://huggingface.co/camstack/camstack-models/resolve/main/objectDetection/yolov9/onnx/camstack-yolov9m-640.onnx", + "imgsz": 640, + "floors": {"person": 0.6, "vehicle": 0.4, "animal": 0.55}, + }, + "dataset": {"versionId": "v1", "annotationHash": "abc", "sha256": "0" * 64}, + "split": {"holdoutCameras": [], "holdoutLastDays": 0, "valFraction": 0.15, "seed": 1}, + "hyper": {"epochs": 2, "batch": 4, "lr0": 0.002, "freeze": 10, "patience": 10, "seed": 1}, + "replay": {"source": "coco2017", "count": 0}, + "exports": ["onnx", "openvino-fp16", "openvino-int8", "coreml-fp16"], + "calibration": {"maxFrames": 300}, + "floors": {"minSupport": 20, "clamp": 0.15}, + "int8Gate": {"maxMap50Drop": 0.02}, + "catalog": {"id": "custom-yolov9m-640-ab12cd34", "name": "Custom YOLOv9 M"}, +} + + +@pytest.fixture +def job_dict() -> dict: + return copy.deepcopy(JOB) + + +def _jpeg() -> bytes: + buf = io.BytesIO() + Image.new("RGB", (64, 48), (90, 120, 150)).save(buf, "JPEG") + return buf.getvalue() + + +def subject(macro: str, label: str | None = None, x: float = 0.1, kind: str = "subject") -> dict: + a = { + "id": f"a-{macro}-{x}-{kind}", "macroClass": macro, "kind": kind, "source": "operator", + "bbox": {"x": x, "y": 0.2, "w": 0.3, "h": 0.4}, "derived": {}, + } + if label is not None: + a["label"] = label + return a + + +def frame(fid: str, device: int, day: int, annotations: list[dict], complete: bool = True, night: bool = False) -> dict: + return { + "frameId": fid, "path": f"frames/{device}/t{fid}/{fid}.jpg", "deviceId": device, + "origin": "track", "trackId": f"t{fid}", "collectionIds": [], "width": 64, "height": 48, + "sizeBytes": 100, "sourceKind": "keyFrame", "copiedAt": T0, "capturedAt": T0 + day * DAY, + "complete": complete, "lighting": {"condition": "night" if night else "day", "night": night, "source": "clock"}, + "annotations": annotations, + } + + +def write_export(path: Path, frames: list[dict], schema: str = "camstack.retrain-export/v3") -> str: + manifest = { + "schema": schema, "layout": "frames", "generatedAt": T0, "batchId": "b1", + "frameCount": len(frames), "annotationCount": 0, "modelErrorCount": 0, + "classifierCropPadding": 0.15, "frames": frames, + } + data = _jpeg() + with tarfile.open(path, "w") as tar: + body = json.dumps(manifest).encode() + info = tarfile.TarInfo("annotations.json") + info.size = len(body) + tar.addfile(info, io.BytesIO(body)) + for f in frames: + info = tarfile.TarInfo(f["path"]) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return hashlib.sha256(path.read_bytes()).hexdigest() + + +@pytest.fixture +def make_export(tmp_path: Path) -> Callable[..., tuple[Path, str]]: + def build(frames: list[dict], schema: str = "camstack.retrain-export/v3") -> tuple[Path, str]: + path = tmp_path / "dataset.tar" + return path, write_export(path, frames, schema) + + return build diff --git a/tests/fakes.py b/tests/fakes.py new file mode 100644 index 0000000000000000000000000000000000000000..6bba8a0671e9871c28244493c3598f614dd753ad --- /dev/null +++ b/tests/fakes.py @@ -0,0 +1,71 @@ +"""Fakes for the heavy toolchain: no torch, Ultralytics or OpenVINO needed.""" +from __future__ import annotations + +from pathlib import Path + +from camstack_train._vendor.camstack_eval.metrics import Detection +from camstack_train.manifest import MLPACKAGE_FILES +from camstack_train.train_det import TrainOutcome + + +def fake_trainer(job, data_yaml, work, cache, emitter): + best = work / "runs" / "train" / "weights" / "best.pt" + best.parent.mkdir(parents=True, exist_ok=True) + best.write_bytes(b"weights") + assert data_yaml.is_file() + emitter.epoch(1, job.hyper.epochs, 1.0, 0.4, 0.2, 10.0) + return TrainOutcome(best, 1, 1, 0.4, 0.2, 3.0) + + +class FakeToolchain: + def __init__(self, fail: tuple[str, ...] = ()) -> None: + self.fail = fail + self.calls: list[str] = [] + self.calib: list[Path] = [] + + def _maybe_fail(self, step: str) -> None: + self.calls.append(step) + if step in self.fail: + raise RuntimeError(f"{step} exploded") + + def onnx(self, pt, out, imgsz): + self._maybe_fail("onnx") + out.write_bytes(b"onnx") + + def openvino(self, onnx, xml, fp16): + self._maybe_fail("openvino") + xml.write_bytes(b"") + xml.with_suffix(".bin").write_bytes(b"fp16") + + def openvino_int8(self, onnx, xml, calib, imgsz): + self._maybe_fail("openvino-int8") + self.calib = list(calib) + xml.write_bytes(b"") + xml.with_suffix(".bin").write_bytes(b"int8") + + def coreml(self, pt, out, imgsz): + self._maybe_fail("coreml") + for m in MLPACKAGE_FILES: + p = out / m + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(b"cm") + + +def detector_factory(int8_score: float = 0.9): + """Every model finds the one person each fixture frame holds; the INT8 IR at ``int8_score``.""" + + def make(path: str, size: int): + score = int8_score if path.endswith("-int8.xml") else 0.9 + + def detect(_jpeg: bytes): + return [Detection("person", score, (0.1, 0.2, 0.3, 0.4))] if score > 0 else [] + + return detect + + return make + + +def fake_fetch(url: str) -> bytes: + if url.endswith(".onnx"): + return b"baseline-onnx" + raise AssertionError(f"unexpected fetch {url}") diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..5daa29df38bae74a5465c7d742b0ec69e84ccc75 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import io +import json + +from camstack_train.cli import main +from camstack_train.events import Emitter +from camstack_train.run import Deps + +from .conftest import frame, subject +from .fakes import FakeToolchain, detector_factory, fake_fetch, fake_trainer + + +def test_bad_spec_exits_64_with_a_failed_result(tmp_path, job_dict): + job_dict["schema"] = "camstack.training/v0" + (tmp_path / "job.json").write_text(json.dumps(job_dict)) + buf = io.StringIO() + code = main(["run", "--job", str(tmp_path / "job.json"), "--dataset", str(tmp_path / "d.tar"), + "--out", str(tmp_path / "out"), "--cache", str(tmp_path / "c")], deps=Deps(emitter=Emitter(buf))) + assert code == 64 + result = json.loads((tmp_path / "out" / "result.json").read_text()) + assert result["status"] == "failed" and result["jobKey"] == job_dict["jobKey"] + assert result["error"]["code"] == "unknown-schema" + + +def test_run_ok_exits_0(tmp_path, job_dict, make_export): + path, digest = make_export([frame(f"a{i}", 1, i, [subject("person")]) for i in range(3)]) + job_dict["dataset"]["sha256"] = digest + (tmp_path / "job.json").write_text(json.dumps(job_dict)) + deps = Deps(fetch=fake_fetch, trainer=fake_trainer, toolchain=FakeToolchain(), + detector=detector_factory(), emitter=Emitter(io.StringIO())) + code = main(["run", "--job", str(tmp_path / "job.json"), "--dataset", str(path), "--out", str(tmp_path / "out"), + "--cache", str(tmp_path / "c"), "--work", str(tmp_path / "w")], deps=deps) + assert code == 0 + + +def test_dataset_command(tmp_path, make_export, capsys): + path, _ = make_export([frame("a", 1, 0, [subject("person")]), frame("b", 1, 0, [subject("animal")])]) + assert main(["dataset", "--export", str(path), "--out", str(tmp_path / "o")]) == 0 + assert "1 frames, 1 boxes" in capsys.readouterr().out diff --git a/tests/test_contract.py b/tests/test_contract.py new file mode 100644 index 0000000000000000000000000000000000000000..6d663b5c426489e58726f781ced31b0d580f7d2e --- /dev/null +++ b/tests/test_contract.py @@ -0,0 +1,35 @@ +"""result.json validates against the published model; the schemas are current.""" +from __future__ import annotations + +import io +import json +import subprocess +import sys +from pathlib import Path + +from camstack_train.events import Emitter +from camstack_train.manifest import failed_result +from camstack_train.result_model import TrainingResult +from camstack_train.run import Deps, run_job + +from .fakes import FakeToolchain, detector_factory, fake_fetch, fake_trainer +from .test_run import setup + +ROOT = Path(__file__).resolve().parents[1] + + +def test_succeeded_result_matches_the_model(tmp_path, make_export, job_dict): + job, paths = setup(tmp_path, make_export, job_dict) + deps = Deps(fetch=fake_fetch, trainer=fake_trainer, toolchain=FakeToolchain(fail=("coreml",)), + detector=detector_factory(), emitter=Emitter(io.StringIO())) + assert run_job(job, paths, deps) == 0 + TrainingResult.model_validate(json.loads((paths.out / "result.json").read_text())) + + +def test_failed_result_matches_the_model(): + TrainingResult.model_validate(failed_result("k", {"stage": "prepare", "code": "bad-spec", "message": "m"})) + + +def test_checked_in_schemas_are_current(): + proc = subprocess.run([sys.executable, str(ROOT / "tools" / "gen_schemas.py"), "--check"], capture_output=True, text=True) + assert proc.returncode == 0, proc.stdout diff --git a/tests/test_dataset.py b/tests/test_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..f676e845836dbd60dff590f6869ece80ba8ea572 --- /dev/null +++ b/tests/test_dataset.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import io +import tarfile + +import pytest + +from camstack_train.dataset import ( + coco_80, coco_index_for, extract_archive, frames_from_manifest, verify_archive, write_layout, +) +from camstack_train.errors import EXIT_DATASET, TrainerError + +from .conftest import frame, subject + + +def test_coco_mapping_is_honest(): + names = coco_80() + assert coco_index_for("person", None, names) == 0 + assert coco_index_for("vehicle", "Car", names) == names.index("car") + assert coco_index_for("animal", "dog", names) == names.index("dog") + assert coco_index_for("vehicle", None, names) is None # no class to give it + assert coco_index_for("animal", "car", names) is None # label of another macro + + +def test_boxes_become_yolo_lines_and_model_errors_are_not_positives(make_export, tmp_path): + frames = [ + frame("f1", 615, 0, [subject("person"), subject("vehicle", "car", x=0.5), subject("person", kind="model_error", x=0.6)]), + frame("f2", 615, 0, []), # complete, empty: a hard negative + frame("f3", 617, 0, [subject("animal")]), # animal without a COCO label: unmappable + frame("f4", 617, 0, [subject("package")], complete=False), # incomplete, no detector subject… + frame("f5", 617, 0, [], complete=False), # incomplete, nothing: never trains + ] + path, digest = make_export(frames) + verify_archive(path, digest) + manifest = extract_archive(path, tmp_path / "x") + parsed = {f.frame_id: f for f in frames_from_manifest(manifest)} + f1 = parsed["f1"] + assert [b.cls_index for b in f1.labels] == [0, coco_80().index("car")] + assert f1.model_errors == 1 and f1.trainable + assert parsed["f2"].trainable and parsed["f2"].labels == () + assert not parsed["f3"].trainable and parsed["f3"].unmappable == 1 + assert not parsed["f4"].trainable # a package box is not a root-detector subject + assert not parsed["f5"].trainable + box = f1.labels[0] + assert (round(box.cx, 3), round(box.cy, 3), round(box.w, 3), round(box.h, 3)) == (0.25, 0.4, 0.3, 0.4) + + counts = write_layout({"train": [f for f in parsed.values() if f.trainable]}, tmp_path / "x", tmp_path / "yolo") + assert counts.frames["train"] == 2 + label_files = sorted((tmp_path / "yolo" / "labels" / "train").glob("*.txt")) + assert [p.name for p in label_files] == ["000001.txt", "000002.txt"] # neutral names + assert label_files[0].read_text().splitlines()[0].startswith("0 0.250000 0.400000") + assert label_files[1].read_text() == "" + + +def test_incomplete_frame_with_a_box_trains(make_export, tmp_path): + path, _ = make_export([frame("f1", 1, 0, [subject("person")], complete=False)]) + f = frames_from_manifest(extract_archive(path, tmp_path / "x"))[0] + assert f.trainable and not f.complete + + +def test_out_of_frame_box_is_clamped_or_dropped(): + names = coco_80() + from camstack_train.dataset import frame_from_entry + + entry = frame("f", 1, 0, [ + {**subject("person"), "bbox": {"x": -0.2, "y": 0.0, "w": 0.4, "h": 0.5}}, + {**subject("person"), "bbox": {"x": 1.2, "y": 0.0, "w": 0.4, "h": 0.5}}, + ]) + f = frame_from_entry(entry, names) + assert len(f.labels) == 1 and round(f.labels[0].w, 3) == 0.2 + + +def test_sha256_mismatch_is_a_dataset_error(make_export): + path, _ = make_export([frame("f1", 1, 0, [])]) + with pytest.raises(TrainerError) as err: + verify_archive(path, "1" * 64) + assert err.value.exit_code == EXIT_DATASET and err.value.code == "dataset-sha256" + + +def test_unknown_export_schema_refused(make_export, tmp_path): + path, _ = make_export([frame("f1", 1, 0, [])], schema="camstack.retrain-export/v9") + with pytest.raises(TrainerError) as err: + extract_archive(path, tmp_path / "x") + assert err.value.code == "dataset-schema" + + +def test_extraction_skips_unsafe_members(tmp_path, make_export): + path, _ = make_export([frame("f1", 1, 0, [])]) + with tarfile.open(path, "a") as tar: + info = tarfile.TarInfo("../evil.jpg") + info.size = 1 + tar.addfile(info, io.BytesIO(b"x")) + extract_archive(path, tmp_path / "x") + assert not (tmp_path / "evil.jpg").exists() diff --git a/tests/test_events.py b/tests/test_events.py new file mode 100644 index 0000000000000000000000000000000000000000..ad2e79116f9d08f329fef5f7c39b4c4c21626c0c --- /dev/null +++ b/tests/test_events.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import io +import json + +import pytest + +from camstack_train.events import PREFIX, Emitter, parse_line + + +def lines(buf: io.StringIO) -> list[dict]: + return [json.loads(l[len(PREFIX):]) for l in buf.getvalue().splitlines()] + + +def test_every_event_is_one_prefixed_compact_line(): + buf = io.StringIO() + em = Emitter(buf, clock=lambda: 12.5) + em.stage("train") + em.epoch(3, 40, 1.2, 0.5, 0.3, 90.0) + em.done("succeeded", "f" * 64) + raw = buf.getvalue().splitlines() + assert all(l.startswith(PREFIX) for l in raw) + events = lines(buf) + assert [e["type"] for e in events] == ["stage", "epoch", "done"] + assert all(e["v"] == 1 and e["ts"] == 12500 for e in events) + assert events[1] == {"v": 1, "ts": 12500, "type": "epoch", "epoch": 3, "epochs": 40, + "boxLoss": 1.2, "map50": 0.5, "map50_95": 0.3, "etaSec": 90.0} + assert events[2]["resultSha256"] == "f" * 64 + assert all(", " not in l and ": " not in l for l in raw) + + +def test_unknown_stage_is_a_bug(): + with pytest.raises(ValueError): + Emitter(io.StringIO()).stage("bake") + + +def test_parse_line_ignores_other_output(): + assert parse_line("Epoch 1/40 ...") is None + assert parse_line(PREFIX + "{not json") is None + assert parse_line(PREFIX + '{"v":1,"type":"stage"}') == {"v": 1, "type": "stage"} diff --git a/tests/test_export.py b/tests/test_export.py new file mode 100644 index 0000000000000000000000000000000000000000..9ccb30ae60efd1a25bec9b7ead0fd2d736b6b0b9 --- /dev/null +++ b/tests/test_export.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import pytest + +from camstack_train.errors import TrainerError +from camstack_train.export import artefact_names, calibration_frames, export_all, remove_export + +from .fakes import FakeToolchain + +CID = "custom-x" +ALL = ["onnx", "openvino-fp16", "openvino-int8", "coreml-fp16"] + + +def run(tmp_path, toolchain, exports=ALL, calib=True): + pt = tmp_path / "best.pt" + pt.write_bytes(b"w") + images = tmp_path / "train" + images.mkdir() + for i in range(5): + (images / f"{i:06d}.jpg").write_bytes(b"j") + frames = calibration_frames(images, 3) if calib else [] + return export_all(exports, pt, CID, 640, frames, tmp_path / "stage", toolchain) + + +def test_every_format_lands_under_its_flat_name(tmp_path): + tc = FakeToolchain() + out = run(tmp_path, tc) + assert out.produced == ALL and out.dropped == [] + stage = tmp_path / "stage" + for name in artefact_names(CID).values(): + assert (stage / name).exists(), name + assert (stage / f"{CID}-int8.bin").is_file() + assert [p.name for p in tc.calib] == ["000000.jpg", "000001.jpg", "000002.jpg"] + + +def test_failing_coreml_drops_only_coreml(tmp_path): + out = run(tmp_path, FakeToolchain(fail=("coreml",))) + assert "coreml-fp16" not in out.produced + assert out.dropped == [{"export": "coreml-fp16", "reason": "coreml exploded"}] + assert "openvino-int8" in out.produced + + +def test_onnx_failure_fails_the_job(tmp_path): + with pytest.raises(TrainerError) as err: + run(tmp_path, FakeToolchain(fail=("onnx",))) + assert err.value.code == "export-onnx" and err.value.exit_code == 70 + + +def test_int8_without_calibration_frames_is_dropped(tmp_path): + out = run(tmp_path, FakeToolchain(), calib=False) + assert {"export": "openvino-int8", "reason": "no calibration frames in the train split"} in out.dropped + + +def test_only_requested_exports_run(tmp_path): + tc = FakeToolchain() + run(tmp_path, tc, exports=["onnx"]) + assert tc.calls == ["onnx"] + + +def test_remove_export_deletes_xml_and_bin(tmp_path): + run(tmp_path, FakeToolchain()) + remove_export(tmp_path / "stage", CID, "openvino-int8") + assert not (tmp_path / "stage" / f"{CID}-int8.xml").exists() + assert not (tmp_path / "stage" / f"{CID}-int8.bin").exists() + assert (tmp_path / "stage" / f"{CID}-fp16.bin").exists() diff --git a/tests/test_floors.py b/tests/test_floors.py new file mode 100644 index 0000000000000000000000000000000000000000..e88f80cc68d16c8caf71b615d2cea4c222d5a092 --- /dev/null +++ b/tests/test_floors.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from camstack_train._vendor.camstack_eval.floors import FloorPolicy, ScoredDetection +from camstack_train.floors import floors_block, floors_from_report, propose + +POLICY = FloorPolicy(min_support=20, clamp=0.15) +BASE = {"person": 0.6, "vehicle": 0.4, "animal": 0.55} + + +def fps(*scores: float) -> list[ScoredDetection]: + return [ScoredDetection(s, False) for s in scores] + + +def test_lowest_floor_within_the_baseline_fp_budget(): + # baseline: 1 FP at or above its floor 0.6; the candidate makes FPs at 0.52 and 0.66 + baseline = fps(0.62, 0.3) + candidate = fps(0.52, 0.66) + p = propose(baseline, 0.6, candidate, positives=30, policy=POLICY) + assert p.reason == "fp-budget" and p.value == 0.55 and p.fp_budget == 1 + + +def test_low_support_inherits_the_baseline(): + p = propose(fps(0.9), 0.6, fps(0.2), positives=8, policy=POLICY) + assert (p.value, p.reason) == (0.6, "low-support") + + +def test_clamped_to_plus_minus_clamp(): + # a candidate with no false positives would fit the grid minimum 0.25; clamp to 0.45 + p = propose(fps(), 0.6, fps(), positives=50, policy=POLICY) + assert (p.value, p.reason, p.fitted) == (0.45, "clamped", 0.25) + # no grid floor meets the budget → the top of the clamp band + q = propose(fps(), 0.6, fps(0.99), positives=50, policy=POLICY) + assert (q.value, q.reason) == (0.75, "clamped") + + +def report(person: dict) -> dict: + fit = {"value": 0.4, "reason": "fp-budget", "fpBudget": 2, "fitted": 0.4} + return {"frames": 10, "classes": { + "person": {"positives": person.get("positives", 30), "proposedFloor": person}, + "vehicle": {"positives": 40, "proposedFloor": fit}, + "animal": {"positives": 5, "proposedFloor": {"value": 0.55, "reason": "low-support", "fpBudget": 0, "fitted": None}}, + }} + + +def test_report_to_floors_with_notes(): + floors, notes = floors_from_report(report({"value": 0.5, "reason": "fp-budget", "fpBudget": 1, "fitted": 0.5}), BASE, POLICY) + assert floors == {"person": 0.5, "vehicle": 0.4, "animal": 0.55} + assert notes == ["animal: 5 holdout positives < 20, inherits 0.55"] + + +def test_no_holdout_inherits_everything(): + floors, notes = floors_from_report(None, BASE, POLICY) + assert floors == BASE and "inherits the baseline" in notes[0] + + +def test_floors_block_int8_only_when_kept(): + r = report({"value": 0.5, "reason": "fp-budget", "fpBudget": 1, "fitted": 0.5}) + block = floors_block(r, r, BASE, POLICY, int8_present=False) + assert block["candidateInt8"] is None and block["baseline"] == BASE + block = floors_block(r, r, BASE, POLICY, int8_present=True) + assert block["candidateInt8"]["person"] == 0.5 + assert any(n.startswith("int8 animal") for n in block["notes"]) diff --git a/tests/test_manifest.py b/tests/test_manifest.py new file mode 100644 index 0000000000000000000000000000000000000000..c009de47e63e276ddbf44d1baa4aafa7d56f3a51 --- /dev/null +++ b/tests/test_manifest.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import hashlib +import tarfile + +from camstack_train.manifest import MLPACKAGE_FILES, catalog_draft, file_rows, write_bundle +from camstack_train.spec import parse_job + +CID = "custom-yolov9m-640-ab12cd34" + + +def stage(tmp_path, with_int8=True, with_coreml=True): + s = tmp_path / "artifacts" + s.mkdir() + (s / f"{CID}.pt").write_bytes(b"pt") + (s / f"{CID}.onnx").write_bytes(b"onnx") + (s / f"{CID}-fp16.xml").write_bytes(b"") + (s / f"{CID}-fp16.bin").write_bytes(b"fp16w") + if with_int8: + (s / f"{CID}-int8.xml").write_bytes(b"") + (s / f"{CID}-int8.bin").write_bytes(b"int8w") + if with_coreml: + for m in MLPACKAGE_FILES: + p = s / f"{CID}.mlpackage" / m + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(m.encode()) + return s + + +def sha(b: bytes) -> str: + return hashlib.sha256(b).hexdigest() + + +def test_rows_classify_every_file(tmp_path): + rows = {r["path"]: r for r in file_rows(stage(tmp_path), CID)} + assert rows[f"{CID}.onnx"] == {"path": f"{CID}.onnx", "bytes": 4, "sha256": sha(b"onnx"), "format": "onnx", "variant": "base"} + assert rows[f"{CID}-int8.bin"]["variant"] == "int8" and rows[f"{CID}-int8.bin"]["format"] == "openvino" + assert rows[f"{CID}-fp16.bin"]["variant"] == "base" + assert rows[f"{CID}.mlpackage/Data/com.apple.CoreML/weights/weight.bin"]["format"] == "coreml" + assert rows[f"{CID}.pt"]["format"] == "pt" + assert len(rows) == 9 + + +def test_bundle_holds_exactly_the_rows(tmp_path): + s = stage(tmp_path) + rows = file_rows(s, CID) + bundle = write_bundle(s, rows, tmp_path / "out" / "bundle.tar") + assert bundle["path"] == "bundle.tar" and bundle["sha256"] == sha((tmp_path / "out" / "bundle.tar").read_bytes()) + with tarfile.open(tmp_path / "out" / "bundle.tar") as tar: + assert sorted(tar.getnames()) == sorted(r["path"] for r in rows) + again = write_bundle(s, rows, tmp_path / "out2" / "bundle.tar") + assert again["sha256"] == bundle["sha256"] # deterministic + + +def test_catalog_draft_formats_and_revisions(tmp_path, job_dict): + job = parse_job(job_dict) + rows = file_rows(stage(tmp_path), CID) + floors = {"candidate": {"person": 0.5}, "candidateInt8": {"person": 0.45}} + draft = catalog_draft(job, rows, floors) + base = draft["base"] + assert base["id"] == CID and base["defaultClassFloors"] == {"person": 0.5} + assert base["inputSize"] == {"width": 640, "height": 640} and base["preprocessMode"] == "letterbox" + assert base["formats"]["onnx"] == {"url": f"{CID}.onnx", "sizeMB": 0.0, "revision": sha(b"onnx")} + assert base["formats"]["openvino"]["url"] == f"{CID}-fp16.xml" + assert base["formats"]["openvino"]["files"] == [f"{CID}-fp16.bin"] + assert base["formats"]["openvino"]["revision"] == sha(b"fp16w") + cm = base["formats"]["coreml"] + assert cm["url"] == f"{CID}.mlpackage" and cm["isDirectory"] and cm["files"] == list(MLPACKAGE_FILES) + assert cm["revision"] == sha(b"Data/com.apple.CoreML/weights/weight.bin") + assert draft["int8"]["id"] == f"{CID}-int8" + assert list(draft["int8"]["formats"]) == ["openvino"] + assert draft["int8"]["defaultClassFloors"] == {"person": 0.45} + + +def test_draft_without_int8_or_coreml(tmp_path, job_dict): + job = parse_job(job_dict) + rows = file_rows(stage(tmp_path, with_int8=False, with_coreml=False), CID) + draft = catalog_draft(job, rows, {"candidate": {}, "candidateInt8": None}) + assert draft["int8"] is None and "coreml" not in draft["base"]["formats"] diff --git a/tests/test_replay.py b/tests/test_replay.py new file mode 100644 index 0000000000000000000000000000000000000000..a0f9c82599fcbffabb4a2e48ad17162c48a95013 --- /dev/null +++ b/tests/test_replay.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import io +import json +import zipfile + +from camstack_train.dataset import coco_80 +from camstack_train.replay import ANNOTATIONS_URL, prepare_replay, select_images + +ANN = { + "categories": [{"id": 1, "name": "person"}, {"id": 3, "name": "car"}, {"id": 44, "name": "bottle"}], + "images": [ + {"id": 1, "file_name": "000000000003.jpg", "width": 100, "height": 50}, + {"id": 2, "file_name": "000000000001.jpg", "width": 100, "height": 50}, + {"id": 3, "file_name": "000000000002.jpg", "width": 100, "height": 50}, + ], + "annotations": [ + {"image_id": 1, "category_id": 1, "bbox": [10, 5, 20, 10], "iscrowd": 0}, + {"image_id": 1, "category_id": 44, "bbox": [0, 0, 10, 10], "iscrowd": 0}, + {"image_id": 2, "category_id": 3, "bbox": [0, 0, 50, 25], "iscrowd": 0}, + {"image_id": 3, "category_id": 44, "bbox": [0, 0, 10, 10], "iscrowd": 0}, + ], +} + + +def fetcher(calls: list[str]): + def fetch(url: str) -> bytes: + calls.append(url) + if url == ANNOTATIONS_URL: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as z: + z.writestr("annotations/instances_val2017.json", json.dumps(ANN)) + return buf.getvalue() + return b"jpeg" + + return fetch + + +def test_pinned_slice_is_detector_images_in_name_order(): + assert [im["file_name"] for im in select_images(ANN, 10)] == ["000000000001.jpg", "000000000003.jpg"] + assert len(select_images(ANN, 1)) == 1 + + +def test_replay_writes_all_80_classes_and_caches(tmp_path): + calls: list[str] = [] + n = prepare_replay(5, tmp_path / "cache", tmp_path / "yolo", coco_80(), fetcher(calls)) + assert n == 2 + label = (tmp_path / "yolo" / "labels" / "replay" / "000000000003.txt").read_text().splitlines() + assert label[0] == "0 0.200000 0.200000 0.200000 0.200000" + assert label[1].startswith(f"{coco_80().index('bottle')} ") # the head is kept whole + first = len(calls) + prepare_replay(5, tmp_path / "cache", tmp_path / "yolo2", coco_80(), fetcher(calls)) + assert len(calls) == first # served from the cache volume + + +def test_zero_count_fetches_nothing(tmp_path): + calls: list[str] = [] + assert prepare_replay(0, tmp_path, tmp_path, coco_80(), fetcher(calls)) == 0 and calls == [] diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 0000000000000000000000000000000000000000..2c78ce46f9138e8312e38f24d3cfd919f0d8dca6 --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import io +import json + +from camstack_train.events import PREFIX, Emitter +from camstack_train.run import Deps, Paths, run_job +from camstack_train.spec import parse_job + +from .conftest import frame, subject +from .fakes import FakeToolchain, detector_factory, fake_fetch, fake_trainer + + +def setup(tmp_path, make_export, job_dict, holdout=(2,)): + frames = [frame(f"a{i}", 1, i, [subject("person")]) for i in range(4)] + frames += [frame(f"b{i}", 2, i, [subject("person")]) for i in range(3)] + path, digest = make_export(frames) + job_dict["dataset"]["sha256"] = digest + job_dict["split"]["holdoutCameras"] = list(holdout) + job_dict["floors"]["minSupport"] = 2 + paths = Paths(job=tmp_path / "job.json", dataset=path, out=tmp_path / "out", cache=tmp_path / "cache", work=tmp_path / "work") + return parse_job(job_dict), paths + + +def events(buf: io.StringIO) -> list[dict]: + return [json.loads(l[len(PREFIX):]) for l in buf.getvalue().splitlines() if l.startswith(PREFIX)] + + +def deps(buf, **kw) -> Deps: + return Deps(fetch=fake_fetch, trainer=fake_trainer, toolchain=kw.get("toolchain", FakeToolchain()), + detector=kw.get("detector", detector_factory()), emitter=Emitter(buf)) + + +def test_end_to_end_success(tmp_path, make_export, job_dict): + job, paths = setup(tmp_path, make_export, job_dict) + buf = io.StringIO() + assert run_job(job, paths, deps(buf)) == 0 + result = json.loads((paths.out / "result.json").read_text()) + assert result["schema"] == "camstack.training-result/v1" and result["status"] == "succeeded" + assert result["error"] is None + assert result["dataset"]["frames"]["holdout"] == 3 and result["dataset"]["holdoutCameras"] == [2] + assert result["evaluation"]["holdoutFrames"] == 3 + assert result["evaluation"]["candidate"]["classes"]["person"]["recall"] == 1.0 + assert result["int8"]["kept"] is True + assert result["catalogDraft"]["int8"] is not None + assert {f["path"] for f in result["files"]} >= {"custom-yolov9m-640-ab12cd34.onnx"} + assert result["bundle"]["path"] == "bundle.tar" and (paths.out / "bundle.tar").is_file() + stages = [e["stage"] for e in events(buf) if e["type"] == "stage"] + assert stages == ["prepare", "replay", "train", "export", "eval", "package"] + done = events(buf)[-1] + assert done["type"] == "done" and done["status"] == "succeeded" + import hashlib + assert done["resultSha256"] == hashlib.sha256((paths.out / "result.json").read_bytes()).hexdigest() + + +def test_int8_gate_drops_a_worse_int8(tmp_path, make_export, job_dict): + job, paths = setup(tmp_path, make_export, job_dict) + buf = io.StringIO() + assert run_job(job, paths, deps(buf, detector=detector_factory(int8_score=0.0))) == 0 + result = json.loads((paths.out / "result.json").read_text()) + assert result["int8"]["kept"] is False and result["int8"]["reason"].startswith("mAP50 drop") + assert result["catalogDraft"]["int8"] is None + assert not any(f["variant"] == "int8" for f in result["files"]) + assert any(d["export"] == "openvino-int8" for d in result["exportsDropped"]) + + +def test_no_holdout_inherits_floors(tmp_path, make_export, job_dict): + job, paths = setup(tmp_path, make_export, job_dict, holdout=()) + assert run_job(job, paths, deps(io.StringIO())) == 0 + result = json.loads((paths.out / "result.json").read_text()) + assert result["floors"]["candidate"] == job_dict["baseline"]["floors"] + assert result["evaluation"]["candidate"] is None + + +def test_dataset_failure_writes_a_failed_result(tmp_path, make_export, job_dict): + job, paths = setup(tmp_path, make_export, job_dict) + job = job.model_copy(update={"dataset": job.dataset.model_copy(update={"sha256": "1" * 64})}) + buf = io.StringIO() + assert run_job(job, paths, deps(buf)) == 65 + result = json.loads((paths.out / "result.json").read_text()) + assert result["status"] == "failed" and result["error"]["code"] == "dataset-sha256" + kinds = [e["type"] for e in events(buf)] + assert kinds[-2:] == ["error", "done"] + + +def test_unexpected_crash_is_internal_and_scrubbed(tmp_path, make_export, job_dict): + job, paths = setup(tmp_path, make_export, job_dict) + + def boom(job, data_yaml, work, cache, emitter): + raise RuntimeError(f"cannot read {work}/export/frames/615/t1/x.jpg") + + d = deps(io.StringIO()) + d.trainer = boom + assert run_job(job, paths, d) == 70 + result = json.loads((paths.out / "result.json").read_text()) + assert result["error"]["code"] == "internal" + assert str(paths.work) not in result["error"]["message"] and "" in result["error"]["message"] + assert result["dataset"]["frames"]["train"] > 0 # what was known is kept diff --git a/tests/test_spec.py b/tests/test_spec.py new file mode 100644 index 0000000000000000000000000000000000000000..9630e321d0f1086fd57cd266e79047f78f8774a4 --- /dev/null +++ b/tests/test_spec.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import json + +import pytest + +from camstack_train.errors import EXIT_BAD_SPEC, TrainerError +from camstack_train.spec import load_job, parse_job + + +def test_valid_job_parses(job_dict): + job = parse_job(job_dict) + assert job.job_key == job_dict["jobKey"] + assert job.int8_gate.max_map50_drop == 0.02 + assert job.baseline.onnx_url.startswith("https://") + + +def test_unknown_schema_exits_64(job_dict): + job_dict["schema"] = "camstack.training/v2" + with pytest.raises(TrainerError) as err: + parse_job(job_dict) + assert err.value.exit_code == EXIT_BAD_SPEC + assert err.value.code == "unknown-schema" + + +@pytest.mark.parametrize( + "mutate", + [ + lambda j: j.pop("hyper"), # every field is required: no defaults + lambda j: j.update(extra=1), # unknown keys refused + lambda j: j["catalog"].update(id="../escape"), # file-name safe id + lambda j: j.update(exports=["openvino-fp16"]), # onnx is required + lambda j: j["baseline"]["floors"].pop("animal"), + lambda j: j["dataset"].update(sha256="nothex"), + ], +) +def test_refusals_are_bad_spec(job_dict, mutate): + mutate(job_dict) + with pytest.raises(TrainerError) as err: + parse_job(job_dict) + assert err.value.exit_code == EXIT_BAD_SPEC + + +def test_load_job_reports_unreadable_json(tmp_path): + path = tmp_path / "job.json" + path.write_text("{nope") + with pytest.raises(TrainerError) as err: + load_job(path) + assert err.value.exit_code == EXIT_BAD_SPEC + + +def test_load_job_roundtrip(tmp_path, job_dict): + path = tmp_path / "job.json" + path.write_text(json.dumps(job_dict)) + assert load_job(path).catalog.id == job_dict["catalog"]["id"] diff --git a/tests/test_split.py b/tests/test_split.py new file mode 100644 index 0000000000000000000000000000000000000000..ed20b22e862d935585d4c116661cec9fc58d7c86 --- /dev/null +++ b/tests/test_split.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import pytest + +from camstack_train.dataset import coco_80, frame_from_entry +from camstack_train.errors import TrainerError +from camstack_train.spec import Split +from camstack_train.split import assign + +from .conftest import frame, subject + + +def frames(spec: list[tuple[str, int, int]]): + names = coco_80() + return [frame_from_entry(frame(fid, dev, day, [subject("person")]), names) for fid, dev, day in spec] + + +def policy(**kw) -> Split: + base = {"holdout_cameras": [], "holdout_last_days": 0, "val_fraction": 0.0, "seed": 1} + return Split(**{**base, **kw}) + + +def test_holdout_cameras_are_held_out_whole(): + fs = frames([("a", 1, 0), ("b", 1, 1), ("c", 2, 0), ("d", 3, 0)]) + result = assign(fs, policy(holdout_cameras=[2, 3])) + assert {f.frame_id for f in result.holdout} == {"c", "d"} + assert {f.frame_id for f in result.train} == {"a", "b"} + assert result.holdout_cameras == (2, 3) + + +def test_last_days_per_camera(): + fs = frames([("a", 1, 0), ("b", 1, 5), ("c", 2, 2), ("d", 2, 3), ("e", 2, 4)]) + result = assign(fs, policy(holdout_last_days=2)) + # camera 1: last day 5 → day 5 only (day 4 absent); camera 2: last day 4 → days 3, 4 + assert {f.frame_id for f in result.holdout} == {"b", "d", "e"} + + +def test_val_takes_whole_groups_deterministically(): + spec = [(f"f{d}-{i}", cam, d) for cam in (1, 2) for d in range(5) for i in range(3)] + a = assign(frames(spec), policy(val_fraction=0.2, seed=7)) + b = assign(frames(spec), policy(val_fraction=0.2, seed=7)) + assert [f.frame_id for f in a.val] == [f.frame_id for f in b.val] + groups = {(f.device_id, f.captured_at) for f in a.val} + for f in a.train: + assert (f.device_id, f.captured_at) not in groups # a day never straddles + assert 0 < len(a.val) <= 9 + + +def test_val_never_takes_the_last_train_group(): + result = assign(frames([("a", 1, 0), ("b", 1, 0)]), policy(val_fraction=0.5)) + assert len(result.train) == 2 and result.val == () + + +def test_empty_train_refused(): + with pytest.raises(TrainerError) as err: + assign(frames([("a", 1, 0)]), policy(holdout_cameras=[1])) + assert err.value.code == "no-train-frames" and err.value.exit_code == 65 + + +def test_holdout_eval_is_complete_frames_only(): + names = coco_80() + fs = [ + frame_from_entry(frame("a", 1, 0, [subject("person")]), names), + frame_from_entry(frame("b", 2, 0, [subject("person")], complete=False), names), + frame_from_entry(frame("c", 2, 0, [subject("person")]), names), + ] + result = assign(fs, policy(holdout_cameras=[2])) + assert [f.frame_id for f in result.holdout_eval] == ["c"] diff --git a/tests/test_vendor.py b/tests/test_vendor.py new file mode 100644 index 0000000000000000000000000000000000000000..764f499e2652a7085a051e9c927980e9c1f550c2 --- /dev/null +++ b/tests/test_vendor.py @@ -0,0 +1,25 @@ +"""The vendored harness imports the vendored inference pool, not a repo path.""" +from __future__ import annotations + +from pathlib import Path + +from camstack_train._vendor import PIPELINE_PYTHON, ensure_pipeline_on_path + + +def test_harness_resolves_the_vendored_pool(): + ensure_pipeline_on_path() + from camstack_train._vendor.camstack_eval.pipeline import _pipeline + + pool, postprocessors = _pipeline() + assert Path(pool.__file__).resolve().parent == PIPELINE_PYTHON + assert "yolo" in postprocessors + + +def test_pool_letterbox_is_grey_114(): + ensure_pipeline_on_path() + import inference_pool + from PIL import Image + + arr, scale, pad = inference_pool.letterbox(Image.new("RGB", (64, 32), (0, 0, 0)), 64) + assert arr.shape == (64, 64, 3) and pad == (0, 16) and scale == 1.0 + assert abs(float(arr[0, 0, 0]) - 114 / 255) < 1e-6 diff --git a/tools/gen_schemas.py b/tools/gen_schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..494df20fcc56194a5884f7a23fe3510c1d345a90 --- /dev/null +++ b/tools/gen_schemas.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Regenerate schemas/job.schema.json and schemas/result.schema.json from the +pydantic models. Run after changing spec.py or result_model.py: + + python tools/gen_schemas.py # write + python tools/gen_schemas.py --check # exit 1 when a checked-in schema is stale +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from camstack_train.result_model import TrainingResult # noqa: E402 +from camstack_train.spec import JobSpec # noqa: E402 + +TARGETS = { + "job.schema.json": (JobSpec, "camstack.training/v1"), + "result.schema.json": (TrainingResult, "camstack.training-result/v1"), +} + + +def render(model, schema_id: str) -> str: + schema = model.model_json_schema(by_alias=True) + schema = {"$id": schema_id, "$schema": "https://json-schema.org/draft/2020-12/schema", **schema} + return json.dumps(schema, indent=2, sort_keys=True) + "\n" + + +def main(argv: list[str]) -> int: + check = "--check" in argv + stale = [] + for name, (model, schema_id) in TARGETS.items(): + path = ROOT / "schemas" / name + text = render(model, schema_id) + if check: + if not path.is_file() or path.read_text("utf-8") != text: + stale.append(name) + else: + path.write_text(text, "utf-8") + if stale: + print(f"stale: {', '.join(stale)} — run python tools/gen_schemas.py") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))