diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..e79649499245416c5de5b7c1d9c1e7753bccb4bb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +JiT-H-16-SIM/demo.png filter=lfs diff=lfs merge=lfs -text diff --git a/JiT-B-16-SIM/model_index.json b/JiT-B-16-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..6ae22d786e6a73f7b2c61a207ad9634f9a2fef4e --- /dev/null +++ b/JiT-B-16-SIM/model_index.json @@ -0,0 +1,1017 @@ +{ + "_class_name": [ + "pipeline", + "JiTPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_flow_match_fd", + "FDLossFlowMatchScheduler" + ], + "transformer": [ + "jit_transformer_2d", + "JiTTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + } +} diff --git a/JiT-B-16-SIM/pipeline.py b/JiT-B-16-SIM/pipeline.py index 46a8c53a9d91b043faddc8e9b08b8d7f3a0f963a..164b44e829eeedda7dfae5ee8d456a3252082097 100644 --- a/JiT-B-16-SIM/pipeline.py +++ b/JiT-B-16-SIM/pipeline.py @@ -1,11 +1,23 @@ -"""Hub custom pipeline: JiTPipeline for FD-Loss post-trained JiT checkpoints. - -Uses FD-Loss sampling (legacy time convention, velocity Euler/Heun, t: 1→0). -See libs/FD-Loss-diffusers and scripts/evaluate_released_ckpt.sh (JiT_B preset). -""" +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hub custom pipeline: JiTPipeline for FD-Loss post-trained JiT (legacy time, velocity Euler).""" from __future__ import annotations +import importlib.util +import inspect import json from pathlib import Path from typing import Dict, List, Optional, Tuple, Union @@ -14,114 +26,190 @@ import torch from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput from diffusers.utils.torch_utils import randn_tensor -from scheduling_flow_match_fd import FDLossFlowMatchScheduler - -RECOMMENDED_NOISE_BY_SIZE = { - 256: 1.0, - 512: 2.0, -} +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from pathlib import Path + >>> import torch + >>> from diffusers import DiffusionPipeline + + >>> model_dir = Path("BiliSakura/FD-Loss-diffusers/JiT-B-16-SIM") + >>> pipe = DiffusionPipeline.from_pretrained( + ... str(model_dir), + ... custom_pipeline=str(model_dir / "pipeline.py"), + ... trust_remote_code=True, + ... ) + >>> pipe.to("cuda") + >>> image = pipe(class_labels="golden retriever", num_inference_steps=1, guidance_scale=3.0).images[0] + ``` +""" -RECOMMENDED_CFG_BY_VARIANT = { - "JiT-B": 3.0, - "JiT-L": 2.4, - "JiT-H": 2.2, -} +RECOMMENDED_NOISE_BY_SIZE = {256: 1.0, 512: 2.0} class JiTPipeline(DiffusionPipeline): r""" - Pipeline for FD-Loss post-trained JiT (flow matching, legacy time convention). - - Parameters: - transformer ([`JiTTransformer2DModel`]): - Class-conditioned JiT backbone. - scheduler ([`FDLossFlowMatchScheduler`]): - Flow timesteps from 1 (noise) to 0 (data). - legacy_time_convention (`bool`, *optional*, defaults to `True`): - Flip flow time when passing to the backbone (`t_bb = 1 - t`), as in FD-Loss training. - id2label (`dict[int, str]`, *optional*): - ImageNet class id to English label mapping. + Class-conditional JiT pipeline for FD-Loss checkpoints (1-NFE velocity Euler, legacy time convention). + + Args: + transformer ([`JiTTransformer2DModel`]): JiT backbone. + scheduler ([`FDLossFlowMatchScheduler`], *optional*): Bundled flow-matching scheduler (`t=1→0`). + Accepts `generator` in `step()` for reproducible stochastic sampling. + id2label (`dict[int, str]`, *optional*): ImageNet id → label strings from `model_index.json`. """ model_cpu_offload_seq = "transformer" + _optional_components = ["scheduler"] + + @staticmethod + def prepare_extra_step_kwargs( + scheduler, + generator=None, + eta: float | None = None, + ): + kwargs = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + @staticmethod + def _coerce_scheduler(scheduler, transformer): + """Always load the bundled FD-Loss flow scheduler (ignore model_index diffusers schedulers).""" + variant_path = getattr(transformer.config, "_name_or_path", None) + if variant_path: + scheduler_dir = Path(variant_path).resolve().parent / "scheduler" + module_path = scheduler_dir / "scheduling_flow_match_fd.py" + config_path = scheduler_dir / "scheduler_config.json" + if module_path.is_file() and config_path.is_file(): + spec = importlib.util.spec_from_file_location("scheduling_flow_match_fd", module_path) + if spec is not None and spec.loader is not None: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + scheduler_cls = getattr(module, "FDLossFlowMatchScheduler") + return scheduler_cls.from_pretrained(str(scheduler_dir)) + scheduler_dir = Path(__file__).resolve().parent / "scheduler" + module_path = scheduler_dir / "scheduling_flow_match_fd.py" + config_path = scheduler_dir / "scheduler_config.json" + if module_path.is_file() and config_path.is_file(): + spec = importlib.util.spec_from_file_location("scheduling_flow_match_fd", module_path) + if spec is not None and spec.loader is not None: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + scheduler_cls = getattr(module, "FDLossFlowMatchScheduler") + return scheduler_cls.from_pretrained(str(scheduler_dir)) + raise ValueError("FD-Loss JiT requires a bundled FDLossFlowMatchScheduler under `scheduler/`.") def __init__( self, transformer, scheduler=None, id2label: Optional[Dict[Union[int, str], str]] = None, - legacy_time_convention: bool = True, ): super().__init__() - if scheduler is None: - scheduler = FDLossFlowMatchScheduler() + self.legacy_time_convention = True + scheduler = self._coerce_scheduler(scheduler, transformer) self.register_modules(transformer=transformer, scheduler=scheduler) - self.legacy_time_convention = legacy_time_convention self._id2label = self._normalize_id2label(id2label) self.labels = self._build_label2id(self._id2label) self._labels_loaded_from_model_index = bool(self._id2label) - def _backbone_t(self, t: torch.Tensor) -> torch.Tensor: + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + r"""Map ImageNet English label strings to class ids.""" + self._ensure_labels_loaded() + if isinstance(label, str): + label = [label] + missing = [item for item in label if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Examples: {preview}, ...") + return [self.labels[item] for item in label] + + def _backbone_timestep(self, t: torch.Tensor) -> torch.Tensor: if self.legacy_time_convention: return 1.0 - t return t - def _normalize_class_labels( - self, - class_labels: Union[int, str, List[Union[int, str]]], - ) -> List[int]: - if isinstance(class_labels, int): - return [class_labels] - if isinstance(class_labels, str): - return self.get_label_ids(class_labels) - if class_labels and isinstance(class_labels[0], str): - return self.get_label_ids(class_labels) - return list(class_labels) + def _cfg_interval(self, interval_min: float, interval_max: float, device: torch.device) -> Tuple[float, float]: + low = float(self._backbone_timestep(torch.tensor(interval_min, device=device))) + high = float(self._backbone_timestep(torch.tensor(interval_max, device=device))) + return min(low, high), max(low, high) + + def _flow_time_tensors(self, t: torch.Tensor, sample: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Broadcast scalar flow time and per-sample backbone timesteps for `sample` (B, C, H, W).""" + flow_t = t.reshape(()).to(device=sample.device, dtype=sample.dtype) + t_view = flow_t.reshape(*([1] * sample.ndim)) + t_bb = self._backbone_timestep(flow_t).expand(sample.shape[0]) + return t_view, t_bb - def _forward_velocity( + def _predict_velocity( self, - z: torch.Tensor, + sample: torch.Tensor, t: torch.Tensor, - labels: torch.Tensor, + class_labels: torch.Tensor, guidance_scale: float, - cfg_interval: Optional[Tuple[float, float]], + cfg_interval: Tuple[float, float], t_eps: float, + interpolate_pos_encoding: bool, ) -> torch.Tensor: - t_view = t.reshape(-1, *([1] * (z.ndim - 1))) - t_bb = self._backbone_t(t).flatten().expand(z.shape[0]) + t_view, t_bb = self._flow_time_tensors(t, sample) + num_classes = int(getattr(self.transformer.config, "num_classes", 1000)) x_cond = self.transformer( - z, + sample, timestep=t_bb, - class_labels=labels, + class_labels=class_labels, interpolate_pos_encoding=interpolate_pos_encoding, ).sample - v_cond = (z - x_cond) / t_view.clamp_min(t_eps) + v_cond = (sample - x_cond) / t_view.clamp_min(t_eps) - if guidance_scale == 1.0: + if guidance_scale <= 1.0: return v_cond - null_class = int( - getattr(self.transformer.config, "num_classes", getattr(self.transformer.config, "num_class_embeds", 1000)) - ) - class_null = torch.full_like(labels, null_class) + class_null = torch.full_like(class_labels, num_classes) x_uncond = self.transformer( - z, + sample, timestep=t_bb, class_labels=class_null, interpolate_pos_encoding=interpolate_pos_encoding, ).sample - v_uncond = (z - x_uncond) / t_view.clamp_min(t_eps) - - if cfg_interval is None: - return v_uncond + guidance_scale * (v_cond - v_uncond) + v_uncond = (sample - x_uncond) / t_view.clamp_min(t_eps) low, high = cfg_interval - mask = (t < high) & ((low == 0) | (t > low)) + flow_t = t.reshape(()).to(device=sample.device, dtype=sample.dtype) + mask = (flow_t < high) & ((low == 0) | (flow_t > low)) scale = torch.where( mask, - torch.tensor(guidance_scale, device=z.device, dtype=z.dtype), - torch.tensor(1.0, device=z.device, dtype=z.dtype), + torch.tensor(guidance_scale, device=sample.device, dtype=sample.dtype), + torch.tensor(1.0, device=sample.device, dtype=sample.dtype), ) while scale.ndim < v_cond.ndim: scale = scale.unsqueeze(-1) @@ -133,20 +221,6 @@ class JiTPipeline(DiffusionPipeline): return {} return {int(key): value for key, value in id2label.items()} - @staticmethod - def _read_id2label_from_model_index(variant_path: Optional[str]) -> Dict[int, str]: - if not variant_path: - return {} - variant_dir = Path(variant_path).resolve() - model_index_path = variant_dir / "model_index.json" - if not model_index_path.exists(): - return {} - raw = json.loads(model_index_path.read_text(encoding="utf-8")) - id2label = raw.get("id2label") - if not isinstance(id2label, dict): - return {} - return {int(key): value for key, value in id2label.items()} - @staticmethod def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: label2id: Dict[str, int] = {} @@ -160,112 +234,109 @@ class JiTPipeline(DiffusionPipeline): def _ensure_labels_loaded(self) -> None: if self._labels_loaded_from_model_index: return - loaded = self._read_id2label_from_model_index(getattr(self.config, "_name_or_path", None)) - if loaded: - self._id2label = loaded - self.labels = self._build_label2id(self._id2label) + path = getattr(self.config, "_name_or_path", None) + if path: + model_index_path = Path(path).resolve() / "model_index.json" + if model_index_path.exists(): + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if isinstance(id2label, dict): + self._id2label = {int(k): v for k, v in id2label.items()} + self.labels = self._build_label2id(self._id2label) self._labels_loaded_from_model_index = True - @property - def id2label(self) -> Dict[int, str]: - self._ensure_labels_loaded() - return self._id2label - - def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: - self._ensure_labels_loaded() - label2id = self.labels - if not label2id: - raise ValueError("No English labels loaded. Ensure `id2label` exists in model_index.json.") - if isinstance(label, str): - label = [label] - missing = [item for item in label if item not in label2id] - if missing: - preview = ", ".join(list(label2id.keys())[:8]) - raise ValueError(f"Unknown English label(s): {missing}. Example valid labels: {preview}, ...") - return [label2id[item] for item in label] + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) + return list(class_labels) @torch.inference_mode() def __call__( self, class_labels: Union[int, str, List[Union[int, str]]], - num_inference_steps: int = 1, guidance_scale: float = 3.0, guidance_interval_min: float = 0.1, guidance_interval_max: float = 1.0, - sampling_method: str = "euler", noise_scale: Optional[float] = None, t_eps: float = 5e-2, generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + num_inference_steps: int = 1, height: Optional[int] = None, width: Optional[int] = None, interpolate_pos_encoding: bool = True, output_type: Optional[str] = "pil", return_dict: bool = True, ) -> Union[ImagePipelineOutput, Tuple]: + r""" + Generate images conditioned on ImageNet class ids or English labels. + + Args: + class_labels: Class id(s), label string(s), or mixed list. + guidance_scale: CFG scale (FD-Loss JiT-B default: 3.0). + guidance_interval_min: Lower flow-time bound for CFG (default 0.1). + guidance_interval_max: Upper flow-time bound for CFG (default 1.0). + noise_scale: Initial Gaussian scale (default 1.0 at 256px). + t_eps: Clamp for flow time in velocity denominator. + generator: RNG for latent init. + num_inference_steps: FD-Loss sampling steps (1 = single NFE). + height: Output height (defaults to native `sample_size`). + width: Output width (defaults to native `sample_size`). + interpolate_pos_encoding: Interpolate positional embeddings for non-native sizes. + output_type: `pil`, `np`, or `pt`. + return_dict: Return [`ImagePipelineOutput`] if True. + """ + # 1. Check inputs if num_inference_steps < 1: raise ValueError("num_inference_steps must be >= 1.") if output_type not in {"pil", "np", "pt"}: raise ValueError("output_type must be one of: 'pil', 'np', 'pt'.") - if sampling_method not in {"euler", "heun"}: - raise ValueError("sampling_method must be 'euler' or 'heun'.") + # 2. Call parameters class_label_ids = self._normalize_class_labels(class_labels) batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) image_size = int(self.transformer.config.sample_size) patch_size = int(self.transformer.config.patch_size) height = int(height or image_size) width = int(width or image_size) if height % patch_size != 0 or width % patch_size != 0: - raise ValueError( - f"height and width must be divisible by patch_size={patch_size}. Got {(height, width)}." - ) + raise ValueError(f"height/width must be divisible by patch_size={patch_size}.") channels = int(self.transformer.config.in_channels) - + null_class_val = int(getattr(self.transformer.config, "num_classes", 1000)) if noise_scale is None: noise_scale = RECOMMENDED_NOISE_BY_SIZE.get(max(height, width), 1.0) - z = randn_tensor( - shape=(batch_size, channels, height, width), + # 3. Class labels + labels = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1) + labels = labels.clamp(0, null_class_val - 1) + cfg_interval = self._cfg_interval(guidance_interval_min, guidance_interval_max, labels.device) + + # 4. Latents + latents = randn_tensor( + (batch_size, channels, height, width), generator=generator, device=self._execution_device, dtype=self.transformer.dtype, ) * noise_scale - labels = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1) - null_class_val = int( - getattr(self.transformer.config, "num_classes", getattr(self.transformer.config, "num_class_embeds", 1000)) - ) - labels = labels.clamp(0, null_class_val - 1) + # 5–6. Denoising loop (FD-Loss flow t=1 → 0; generator forwarded to scheduler.step) + self.scheduler.set_timesteps(num_inference_steps, device=self._execution_device) + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + timesteps = self.scheduler.timesteps + + for i in self.progress_bar(range(num_inference_steps)): + t_cur = timesteps[i] + velocity = self._predict_velocity( + latents, t_cur, labels, guidance_scale, cfg_interval, t_eps, interpolate_pos_encoding + ) + latents = self.scheduler.step(velocity, t_cur, latents, **extra_step_kwargs).prev_sample - cfg_interval = [ - float(self._backbone_t(torch.tensor(guidance_interval_min, device=z.device))), - float(self._backbone_t(torch.tensor(guidance_interval_max, device=z.device))), - ] - cfg_interval = (min(cfg_interval), max(cfg_interval)) - - timesteps = self.scheduler.set_timesteps(num_inference_steps, device=self._execution_device) - ts = timesteps.view(-1, *([1] * z.ndim)).expand(-1, batch_size, -1, -1, -1) - - for i in self.progress_bar(range(num_inference_steps - 1)): - t_cur = ts[i] - t_next = ts[i + 1] - if sampling_method == "heun": - dt = t_next - t_cur - v1 = self._forward_velocity(z, t_cur, labels, guidance_scale, cfg_interval, t_eps) - z_mid = z + dt * v1 - v2 = self._forward_velocity(z_mid, t_next, labels, guidance_scale, cfg_interval, t_eps) - z = z + dt * 0.5 * (v1 + v2) - else: - v = self._forward_velocity(z, t_cur, labels, guidance_scale, cfg_interval, t_eps) - z = z + (t_next - t_cur) * v - - if num_inference_steps >= 1: - t_cur = ts[-2] - t_next = ts[-1] - v = self._forward_velocity(z, t_cur, labels, guidance_scale, cfg_interval, t_eps) - z = z + (t_next - t_cur) * v - - images_pt = ((z.float().clamp(-1, 1) + 1.0) / 2.0).cpu() + # 8. Output + images_pt = ((latents.float().clamp(-1, 1) + 1.0) / 2.0).cpu() if output_type == "pt": images = images_pt elif output_type == "np": @@ -274,7 +345,6 @@ class JiTPipeline(DiffusionPipeline): images = self.numpy_to_pil(images_pt.permute(0, 2, 3, 1).numpy()) self.maybe_free_model_hooks() - if not return_dict: return (images,) return ImagePipelineOutput(images=images) diff --git a/JiT-B-16-SIM/scheduler/scheduler_config.json b/JiT-B-16-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fda98c585a0e298556a0a75e5f051b6d915ff9e2 --- /dev/null +++ b/JiT-B-16-SIM/scheduler/scheduler_config.json @@ -0,0 +1,6 @@ +{ + "_class_name": "FDLossFlowMatchScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000, + "stochastic_sampling": false +} diff --git a/JiT-B-16-SIM/scheduler/scheduling_flow_match_fd.py b/JiT-B-16-SIM/scheduler/scheduling_flow_match_fd.py index e3b8490ee454c7bfafdaba837f7df693a8cf9fb1..21872a41d87c934e4dd396e33887b19c1d825382 100644 --- a/JiT-B-16-SIM/scheduler/scheduling_flow_match_fd.py +++ b/JiT-B-16-SIM/scheduler/scheduling_flow_match_fd.py @@ -1,70 +1,110 @@ # Copyright 2026 The FD-Loss Authors. SPDX-License-Identifier: MIT -"""Flow-matching scheduler for JiT, pMF, and iMF sampling.""" +"""Flow-matching scheduler for FD-Loss JiT sampling (t: 1 -> 0).""" + +from __future__ import annotations from dataclasses import dataclass -from typing import Optional +from typing import List, Optional, Tuple, Union -import numpy as np import torch - from diffusers.configuration_utils import ConfigMixin, register_to_config from diffusers.schedulers.scheduling_utils import SchedulerMixin from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor @dataclass class FDLossFlowMatchSchedulerOutput(BaseOutput): - prev_sample: torch.FloatTensor + prev_sample: torch.Tensor class FDLossFlowMatchScheduler(SchedulerMixin, ConfigMixin): - """ - Euler / Heun flow-matching scheduler used by FD-Loss generators. - - Timesteps run from 1 (noise) to 0 (data), matching the training convention in this repo. - """ + """Euler flow-matching scheduler with optional stochastic steps and generator-aware noise.""" - config_name = "scheduler_config.json" order = 1 + config_name = "scheduler_config.json" @register_to_config - def __init__(self, num_train_timesteps: int = 1000): - self.num_train_timesteps = num_train_timesteps - self.timesteps = torch.from_numpy(np.linspace(1.0, 0.0, num_train_timesteps + 1)) + def __init__( + self, + num_train_timesteps: int = 1000, + stochastic_sampling: bool = False, + ): + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None - def set_timesteps(self, num_inference_steps: int, device: Optional[torch.device] = None): - self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device) + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> torch.Tensor: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 return self.timesteps + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.reshape(-1)[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + @staticmethod + def _expand_time(value: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + value = value.to(dtype=sample.dtype, device=sample.device) + while value.ndim < sample.ndim: + value = value.unsqueeze(-1) + return value + def step( self, model_output: torch.Tensor, - timestep: torch.Tensor, + timestep: Union[float, torch.Tensor, None], sample: torch.Tensor, - next_timestep: torch.Tensor, return_dict: bool = True, - ) -> FDLossFlowMatchSchedulerOutput: - dt = next_timestep.reshape(-1, *([1] * (sample.ndim - 1))) - timestep.reshape( - -1, *([1] * (sample.ndim - 1)) - ) - prev_sample = sample + dt * model_output - if not return_dict: - return (prev_sample,) - return FDLossFlowMatchSchedulerOutput(prev_sample=prev_sample) + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[FDLossFlowMatchSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = self._expand_time(t - t_next, sample) + + if self.config.stochastic_sampling: + t_expanded = self._expand_time(t, sample) + t_next_expanded = self._expand_time(t_next, sample) + x0 = sample - t_expanded * model_output + noise = randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + prev_sample = (1.0 - t_next_expanded) * x0 + t_next_expanded * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 - def step_heun( - self, - model_output: torch.Tensor, - next_model_output: torch.Tensor, - timestep: torch.Tensor, - sample: torch.Tensor, - next_timestep: torch.Tensor, - return_dict: bool = True, - ) -> FDLossFlowMatchSchedulerOutput: - dt = next_timestep.reshape(-1, *([1] * (sample.ndim - 1))) - timestep.reshape( - -1, *([1] * (sample.ndim - 1)) - ) - prev_sample = sample + dt * 0.5 * (model_output + next_model_output) if not return_dict: return (prev_sample,) return FDLossFlowMatchSchedulerOutput(prev_sample=prev_sample) diff --git a/JiT-B-16-SIM/transformer/config.json b/JiT-B-16-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f016269fa1e363f9b169209b73e3a202d91dc017 --- /dev/null +++ b/JiT-B-16-SIM/transformer/config.json @@ -0,0 +1,18 @@ +{ + "_class_name": "JiTTransformer2DModel", + "_diffusers_version": "0.36.0", + "attention_dropout": 0.0, + "bottleneck_dim": 128, + "dropout": 0.0, + "hidden_size": 768, + "in_channels": 3, + "in_context_len": 32, + "in_context_start": 4, + "mlp_ratio": 4.0, + "norm_eps": 1e-06, + "num_attention_heads": 12, + "num_classes": 1000, + "num_layers": 12, + "patch_size": 16, + "sample_size": 256 +} diff --git a/JiT-B-16-SIM/transformer/diffusion_pytorch_model.safetensors b/JiT-B-16-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e83ef0550be984bf541b1524d6b95ee7c8d5d374 --- /dev/null +++ b/JiT-B-16-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3bda12cc9aae737af3712d1cdabd87431f6df329acce0e2d760431929794fc3 +size 525298808 diff --git a/JiT-H-16-SIM/demo.png b/JiT-H-16-SIM/demo.png new file mode 100644 index 0000000000000000000000000000000000000000..513f974ca6ea2d2243a08d5971239064abfd9112 --- /dev/null +++ b/JiT-H-16-SIM/demo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03188030ad81ad8a3d5b9ace40731a4c3f8bd540e355e465cf80fe2f13bdc17a +size 155347 diff --git a/JiT-H-16-SIM/model_index.json b/JiT-H-16-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..6ae22d786e6a73f7b2c61a207ad9634f9a2fef4e --- /dev/null +++ b/JiT-H-16-SIM/model_index.json @@ -0,0 +1,1017 @@ +{ + "_class_name": [ + "pipeline", + "JiTPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_flow_match_fd", + "FDLossFlowMatchScheduler" + ], + "transformer": [ + "jit_transformer_2d", + "JiTTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + } +} diff --git a/JiT-H-16-SIM/pipeline.py b/JiT-H-16-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..164b44e829eeedda7dfae5ee8d456a3252082097 --- /dev/null +++ b/JiT-H-16-SIM/pipeline.py @@ -0,0 +1,350 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hub custom pipeline: JiTPipeline for FD-Loss post-trained JiT (legacy time, velocity Euler).""" + +from __future__ import annotations + +import importlib.util +import inspect +import json +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union + +import torch +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from pathlib import Path + >>> import torch + >>> from diffusers import DiffusionPipeline + + >>> model_dir = Path("BiliSakura/FD-Loss-diffusers/JiT-B-16-SIM") + >>> pipe = DiffusionPipeline.from_pretrained( + ... str(model_dir), + ... custom_pipeline=str(model_dir / "pipeline.py"), + ... trust_remote_code=True, + ... ) + >>> pipe.to("cuda") + >>> image = pipe(class_labels="golden retriever", num_inference_steps=1, guidance_scale=3.0).images[0] + ``` +""" + +RECOMMENDED_NOISE_BY_SIZE = {256: 1.0, 512: 2.0} + + +class JiTPipeline(DiffusionPipeline): + r""" + Class-conditional JiT pipeline for FD-Loss checkpoints (1-NFE velocity Euler, legacy time convention). + + Args: + transformer ([`JiTTransformer2DModel`]): JiT backbone. + scheduler ([`FDLossFlowMatchScheduler`], *optional*): Bundled flow-matching scheduler (`t=1→0`). + Accepts `generator` in `step()` for reproducible stochastic sampling. + id2label (`dict[int, str]`, *optional*): ImageNet id → label strings from `model_index.json`. + """ + + model_cpu_offload_seq = "transformer" + _optional_components = ["scheduler"] + + @staticmethod + def prepare_extra_step_kwargs( + scheduler, + generator=None, + eta: float | None = None, + ): + kwargs = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + @staticmethod + def _coerce_scheduler(scheduler, transformer): + """Always load the bundled FD-Loss flow scheduler (ignore model_index diffusers schedulers).""" + variant_path = getattr(transformer.config, "_name_or_path", None) + if variant_path: + scheduler_dir = Path(variant_path).resolve().parent / "scheduler" + module_path = scheduler_dir / "scheduling_flow_match_fd.py" + config_path = scheduler_dir / "scheduler_config.json" + if module_path.is_file() and config_path.is_file(): + spec = importlib.util.spec_from_file_location("scheduling_flow_match_fd", module_path) + if spec is not None and spec.loader is not None: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + scheduler_cls = getattr(module, "FDLossFlowMatchScheduler") + return scheduler_cls.from_pretrained(str(scheduler_dir)) + scheduler_dir = Path(__file__).resolve().parent / "scheduler" + module_path = scheduler_dir / "scheduling_flow_match_fd.py" + config_path = scheduler_dir / "scheduler_config.json" + if module_path.is_file() and config_path.is_file(): + spec = importlib.util.spec_from_file_location("scheduling_flow_match_fd", module_path) + if spec is not None and spec.loader is not None: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + scheduler_cls = getattr(module, "FDLossFlowMatchScheduler") + return scheduler_cls.from_pretrained(str(scheduler_dir)) + raise ValueError("FD-Loss JiT requires a bundled FDLossFlowMatchScheduler under `scheduler/`.") + + def __init__( + self, + transformer, + scheduler=None, + id2label: Optional[Dict[Union[int, str], str]] = None, + ): + super().__init__() + self.legacy_time_convention = True + scheduler = self._coerce_scheduler(scheduler, transformer) + self.register_modules(transformer=transformer, scheduler=scheduler) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + r"""Map ImageNet English label strings to class ids.""" + self._ensure_labels_loaded() + if isinstance(label, str): + label = [label] + missing = [item for item in label if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Examples: {preview}, ...") + return [self.labels[item] for item in label] + + def _backbone_timestep(self, t: torch.Tensor) -> torch.Tensor: + if self.legacy_time_convention: + return 1.0 - t + return t + + def _cfg_interval(self, interval_min: float, interval_max: float, device: torch.device) -> Tuple[float, float]: + low = float(self._backbone_timestep(torch.tensor(interval_min, device=device))) + high = float(self._backbone_timestep(torch.tensor(interval_max, device=device))) + return min(low, high), max(low, high) + + def _flow_time_tensors(self, t: torch.Tensor, sample: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Broadcast scalar flow time and per-sample backbone timesteps for `sample` (B, C, H, W).""" + flow_t = t.reshape(()).to(device=sample.device, dtype=sample.dtype) + t_view = flow_t.reshape(*([1] * sample.ndim)) + t_bb = self._backbone_timestep(flow_t).expand(sample.shape[0]) + return t_view, t_bb + + def _predict_velocity( + self, + sample: torch.Tensor, + t: torch.Tensor, + class_labels: torch.Tensor, + guidance_scale: float, + cfg_interval: Tuple[float, float], + t_eps: float, + interpolate_pos_encoding: bool, + ) -> torch.Tensor: + t_view, t_bb = self._flow_time_tensors(t, sample) + num_classes = int(getattr(self.transformer.config, "num_classes", 1000)) + + x_cond = self.transformer( + sample, + timestep=t_bb, + class_labels=class_labels, + interpolate_pos_encoding=interpolate_pos_encoding, + ).sample + v_cond = (sample - x_cond) / t_view.clamp_min(t_eps) + + if guidance_scale <= 1.0: + return v_cond + + class_null = torch.full_like(class_labels, num_classes) + x_uncond = self.transformer( + sample, + timestep=t_bb, + class_labels=class_null, + interpolate_pos_encoding=interpolate_pos_encoding, + ).sample + v_uncond = (sample - x_uncond) / t_view.clamp_min(t_eps) + + low, high = cfg_interval + flow_t = t.reshape(()).to(device=sample.device, dtype=sample.dtype) + mask = (flow_t < high) & ((low == 0) | (flow_t > low)) + scale = torch.where( + mask, + torch.tensor(guidance_scale, device=sample.device, dtype=sample.dtype), + torch.tensor(1.0, device=sample.device, dtype=sample.dtype), + ) + while scale.ndim < v_cond.ndim: + scale = scale.unsqueeze(-1) + return v_uncond + scale * (v_cond - v_uncond) + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + path = getattr(self.config, "_name_or_path", None) + if path: + model_index_path = Path(path).resolve() / "model_index.json" + if model_index_path.exists(): + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if isinstance(id2label, dict): + self._id2label = {int(k): v for k, v in id2label.items()} + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) + return list(class_labels) + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + guidance_scale: float = 3.0, + guidance_interval_min: float = 0.1, + guidance_interval_max: float = 1.0, + noise_scale: Optional[float] = None, + t_eps: float = 5e-2, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + num_inference_steps: int = 1, + height: Optional[int] = None, + width: Optional[int] = None, + interpolate_pos_encoding: bool = True, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + r""" + Generate images conditioned on ImageNet class ids or English labels. + + Args: + class_labels: Class id(s), label string(s), or mixed list. + guidance_scale: CFG scale (FD-Loss JiT-B default: 3.0). + guidance_interval_min: Lower flow-time bound for CFG (default 0.1). + guidance_interval_max: Upper flow-time bound for CFG (default 1.0). + noise_scale: Initial Gaussian scale (default 1.0 at 256px). + t_eps: Clamp for flow time in velocity denominator. + generator: RNG for latent init. + num_inference_steps: FD-Loss sampling steps (1 = single NFE). + height: Output height (defaults to native `sample_size`). + width: Output width (defaults to native `sample_size`). + interpolate_pos_encoding: Interpolate positional embeddings for non-native sizes. + output_type: `pil`, `np`, or `pt`. + return_dict: Return [`ImagePipelineOutput`] if True. + """ + # 1. Check inputs + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + if output_type not in {"pil", "np", "pt"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt'.") + + # 2. Call parameters + class_label_ids = self._normalize_class_labels(class_labels) + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + image_size = int(self.transformer.config.sample_size) + patch_size = int(self.transformer.config.patch_size) + height = int(height or image_size) + width = int(width or image_size) + if height % patch_size != 0 or width % patch_size != 0: + raise ValueError(f"height/width must be divisible by patch_size={patch_size}.") + channels = int(self.transformer.config.in_channels) + null_class_val = int(getattr(self.transformer.config, "num_classes", 1000)) + if noise_scale is None: + noise_scale = RECOMMENDED_NOISE_BY_SIZE.get(max(height, width), 1.0) + + # 3. Class labels + labels = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1) + labels = labels.clamp(0, null_class_val - 1) + cfg_interval = self._cfg_interval(guidance_interval_min, guidance_interval_max, labels.device) + + # 4. Latents + latents = randn_tensor( + (batch_size, channels, height, width), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) * noise_scale + + # 5–6. Denoising loop (FD-Loss flow t=1 → 0; generator forwarded to scheduler.step) + self.scheduler.set_timesteps(num_inference_steps, device=self._execution_device) + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + timesteps = self.scheduler.timesteps + + for i in self.progress_bar(range(num_inference_steps)): + t_cur = timesteps[i] + velocity = self._predict_velocity( + latents, t_cur, labels, guidance_scale, cfg_interval, t_eps, interpolate_pos_encoding + ) + latents = self.scheduler.step(velocity, t_cur, latents, **extra_step_kwargs).prev_sample + + # 8. Output + images_pt = ((latents.float().clamp(-1, 1) + 1.0) / 2.0).cpu() + if output_type == "pt": + images = images_pt + elif output_type == "np": + images = images_pt.permute(0, 2, 3, 1).numpy() + else: + images = self.numpy_to_pil(images_pt.permute(0, 2, 3, 1).numpy()) + + self.maybe_free_model_hooks() + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) diff --git a/JiT-H-16-SIM/scheduler/__pycache__/scheduling_flow_match_fd.cpython-312.pyc b/JiT-H-16-SIM/scheduler/__pycache__/scheduling_flow_match_fd.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fca81c5f369d03afd5a464e3b7f2c118954a88e9 Binary files /dev/null and b/JiT-H-16-SIM/scheduler/__pycache__/scheduling_flow_match_fd.cpython-312.pyc differ diff --git a/JiT-H-16-SIM/scheduler/scheduler_config.json b/JiT-H-16-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fda98c585a0e298556a0a75e5f051b6d915ff9e2 --- /dev/null +++ b/JiT-H-16-SIM/scheduler/scheduler_config.json @@ -0,0 +1,6 @@ +{ + "_class_name": "FDLossFlowMatchScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000, + "stochastic_sampling": false +} diff --git a/JiT-H-16-SIM/scheduler/scheduling_flow_match_fd.py b/JiT-H-16-SIM/scheduler/scheduling_flow_match_fd.py new file mode 100644 index 0000000000000000000000000000000000000000..21872a41d87c934e4dd396e33887b19c1d825382 --- /dev/null +++ b/JiT-H-16-SIM/scheduler/scheduling_flow_match_fd.py @@ -0,0 +1,110 @@ +# Copyright 2026 The FD-Loss Authors. SPDX-License-Identifier: MIT +"""Flow-matching scheduler for FD-Loss JiT sampling (t: 1 -> 0).""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class FDLossFlowMatchSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class FDLossFlowMatchScheduler(SchedulerMixin, ConfigMixin): + """Euler flow-matching scheduler with optional stochastic steps and generator-aware noise.""" + + order = 1 + config_name = "scheduler_config.json" + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + stochastic_sampling: bool = False, + ): + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> torch.Tensor: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + return self.timesteps + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.reshape(-1)[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + @staticmethod + def _expand_time(value: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + value = value.to(dtype=sample.dtype, device=sample.device) + while value.ndim < sample.ndim: + value = value.unsqueeze(-1) + return value + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[FDLossFlowMatchSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = self._expand_time(t - t_next, sample) + + if self.config.stochastic_sampling: + t_expanded = self._expand_time(t, sample) + t_next_expanded = self._expand_time(t_next, sample) + x0 = sample - t_expanded * model_output + noise = randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + prev_sample = (1.0 - t_next_expanded) * x0 + t_next_expanded * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return FDLossFlowMatchSchedulerOutput(prev_sample=prev_sample) diff --git a/JiT-H-16-SIM/transformer/config.json b/JiT-H-16-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ae4bf474c561e132cd3b5d3336c755fa7ae5bda6 --- /dev/null +++ b/JiT-H-16-SIM/transformer/config.json @@ -0,0 +1,18 @@ +{ + "_class_name": "JiTTransformer2DModel", + "_diffusers_version": "0.36.0", + "attention_dropout": 0.0, + "bottleneck_dim": 256, + "dropout": 0.2, + "hidden_size": 1280, + "in_channels": 3, + "in_context_len": 32, + "in_context_start": 10, + "mlp_ratio": 4.0, + "norm_eps": 1e-06, + "num_attention_heads": 16, + "num_classes": 1000, + "num_layers": 32, + "patch_size": 16, + "sample_size": 256 +} diff --git a/JiT-H-16-SIM/transformer/diffusion_pytorch_model.safetensors b/JiT-H-16-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..48bb351dc96696540ff7f9cc452501e6f87cd538 --- /dev/null +++ b/JiT-H-16-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:901c710d3f7bc2243288f0e995f924c9d5b7425d2611391c66b75f6c52e94350 +size 3811413928 diff --git a/JiT-H-16-SIM/transformer/jit_transformer_2d.py b/JiT-H-16-SIM/transformer/jit_transformer_2d.py new file mode 100644 index 0000000000000000000000000000000000000000..1f14b1fc440e7c3730c295f0a90b883e32c12322 --- /dev/null +++ b/JiT-H-16-SIM/transformer/jit_transformer_2d.py @@ -0,0 +1,559 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm +from diffusers.utils import logging + +logger = logging.get_logger(__name__) + + + +def broadcat(tensors, dim=-1): + num_tensors = len(tensors) + shape_lens = {len(t.shape) for t in tensors} + if len(shape_lens) != 1: + raise ValueError("tensors must all have the same number of dimensions") + shape_len = list(shape_lens)[0] + dim = (dim + shape_len) if dim < 0 else dim + dims = list(zip(*(list(t.shape) for t in tensors))) + expandable_dims = [(i, val) for i, val in enumerate(dims) if i != dim] + + if not all(len(set(t[1])) <= 2 for t in expandable_dims): + raise ValueError("invalid dimensions for broadcastable concatenation") + + max_dims = [(t[0], max(t[1])) for t in expandable_dims] + expanded_dims = [(t[0], (t[1],) * num_tensors) for t in max_dims] + expanded_dims.insert(dim, (dim, dims[dim])) + expandable_shapes = list(zip(*(t[1] for t in expanded_dims))) + tensors = [t[0].expand(*t[1]) for t in zip(tensors, expandable_shapes)] + return torch.cat(tensors, dim=dim) + + +def rotate_half(x): + x = x.view(*x.shape[:-1], x.shape[-1] // 2, 2) + x1, x2 = x.unbind(dim=-1) + x = torch.stack((-x2, x1), dim=-1) + return x.view(*x.shape[:-2], -1) + + +class JiTRotaryEmbedding(nn.Module): + def __init__( + self, + dim, + pt_seq_len=16, + ft_seq_len=None, + custom_freqs=None, + theta=10000, + num_cls_token=0, + ): + super().__init__() + self.dim = dim + self.pt_seq_len = pt_seq_len + self.theta = theta + self.num_cls_token = num_cls_token + self.custom_freqs = custom_freqs + if ft_seq_len is None: + ft_seq_len = pt_seq_len + self._cached_hw = None + cos, sin = self._build_freqs(ft_seq_len, ft_seq_len, device=torch.device("cpu")) + self.register_buffer("freqs_cos", cos, persistent=False) + self.register_buffer("freqs_sin", sin, persistent=False) + self._cached_hw = (ft_seq_len, ft_seq_len) + + def _build_freqs(self, height, width, device): + if self.custom_freqs is not None: + freqs = self.custom_freqs.to(device=device, dtype=torch.float32) + else: + freqs = 1.0 / ( + self.theta ** (torch.arange(0, self.dim, 2, device=device, dtype=torch.float32)[: (self.dim // 2)] / self.dim) + ) + + t_h = torch.arange(height, device=device, dtype=torch.float32) / height * self.pt_seq_len + t_w = torch.arange(width, device=device, dtype=torch.float32) / width * self.pt_seq_len + freqs_h = torch.einsum("..., f -> ... f", t_h, freqs).repeat_interleave(2, dim=-1) + freqs_w = torch.einsum("..., f -> ... f", t_w, freqs).repeat_interleave(2, dim=-1) + freqs_2d = broadcat((freqs_h[:, None, :], freqs_w[None, :, :]), dim=-1) + freqs_flat = freqs_2d.view(-1, freqs_2d.shape[-1]) + cos_img = freqs_flat.cos() + sin_img = freqs_flat.sin() + if self.num_cls_token > 0: + _, dim_freq = cos_img.shape + cos_pad = torch.ones(self.num_cls_token, dim_freq, dtype=cos_img.dtype, device=device) + sin_pad = torch.zeros(self.num_cls_token, dim_freq, dtype=sin_img.dtype, device=device) + cos_img = torch.cat([cos_pad, cos_img], dim=0) + sin_img = torch.cat([sin_pad, sin_img], dim=0) + return cos_img, sin_img + + def forward(self, t, height=None, width=None): + # Applied on (batch, seq_len, heads, head_dim) tensors from attention. + seq_len = t.shape[1] + if height is None or width is None: + image_tokens = seq_len - self.num_cls_token + size = int(image_tokens**0.5) + if size * size != image_tokens: + raise ValueError( + f"Cannot infer square token grid from sequence length {seq_len} with {self.num_cls_token} class tokens." + ) + height = size + width = size + if self._cached_hw != (height, width) or self.freqs_cos.device != t.device: + self.freqs_cos, self.freqs_sin = self._build_freqs(height, width, device=t.device) + self._cached_hw = (height, width) + freqs_cos = self.freqs_cos[:seq_len].to(t.dtype) + freqs_sin = self.freqs_sin[:seq_len].to(t.dtype) + + return t * freqs_cos[:, None, :] + rotate_half(t) * freqs_sin[:, None, :] + + +def modulate(x, shift, scale): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +class JiTPatchEmbed(nn.Module): + """Image to Patch Embedding with Bottleneck""" + + def __init__(self, img_size=224, patch_size=16, in_chans=3, pca_dim=768, embed_dim=768, bias=True): + super().__init__() + img_size = (img_size, img_size) + patch_size = (patch_size, patch_size) + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0]) + + self.proj1 = nn.Conv2d(in_chans, pca_dim, kernel_size=patch_size, stride=patch_size, bias=False) + self.proj2 = nn.Conv2d(pca_dim, embed_dim, kernel_size=1, stride=1, bias=bias) + + def forward(self, x): + x = self.proj2(self.proj1(x)).flatten(2).transpose(1, 2) + return x + + +class JiTTimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + + def __init__(self, hidden_size, frequency_embedding_size=256): + super().__init__() + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=True), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size, bias=True), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t, dim, max_period=10000): + """ + Create sinusoidal timestep embeddings. + """ + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half + ).to(device=t.device) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t, dtype=None): + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + if dtype is not None: + t_freq = t_freq.to(dtype=dtype) + t_emb = self.mlp(t_freq) + return t_emb + + +class JiTLabelEmbedder(nn.Module): + """ + Embeds class labels into vector representations. + """ + + def __init__(self, num_classes, hidden_size): + super().__init__() + self.embedding_table = nn.Embedding(num_classes + 1, hidden_size) + self.num_classes = num_classes + + def forward(self, labels): + embeddings = self.embedding_table(labels) + return embeddings + + +class JiTAttention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=True, qk_norm=True, attn_drop=0.0, proj_drop=0.0, eps=1e-6): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + + self.q_norm = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity() + self.k_norm = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity() + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = attn_drop + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, rope=None, grid_height=None, grid_width=None): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + + q = self.q_norm(q) + k = self.k_norm(k) + + if rope is not None: + q = q.transpose(1, 2) + k = k.transpose(1, 2) + q = rope(q, height=grid_height, width=grid_width) + k = rope(k, height=grid_height, width=grid_width) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + + dropout_p = self.attn_drop if self.training else 0.0 + x = F.scaled_dot_product_attention(q, k, v, dropout_p=dropout_p) + x = x.transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class JiTSwiGLUFFN(nn.Module): + def __init__(self, dim: int, hidden_dim: int, drop=0.0, bias=True) -> None: + super().__init__() + hidden_dim = int(hidden_dim * 2 / 3) + self.w12 = nn.Linear(dim, 2 * hidden_dim, bias=bias) + self.w3 = nn.Linear(hidden_dim, dim, bias=bias) + self.ffn_dropout = nn.Dropout(drop) + + def forward(self, x): + x12 = self.w12(x) + x1, x2 = x12.chunk(2, dim=-1) + hidden = F.silu(x1) * x2 + return self.w3(self.ffn_dropout(hidden)) + + +class JiTBlock(nn.Module): + def __init__(self, hidden_size, num_heads, mlp_ratio=4.0, attn_drop=0.0, proj_drop=0.0, eps=1e-6): + super().__init__() + self.norm1 = RMSNorm(hidden_size, eps=eps) + self.attn = JiTAttention( + hidden_size, + num_heads=num_heads, + qkv_bias=True, + qk_norm=True, + attn_drop=attn_drop, + proj_drop=proj_drop, + eps=eps, + ) + self.norm2 = RMSNorm(hidden_size, eps=eps) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + self.mlp = JiTSwiGLUFFN(hidden_size, mlp_hidden_dim, drop=proj_drop) + + self.act = nn.SiLU() + self.adaLN_modulation = nn.Linear(hidden_size, 6 * hidden_size, bias=True) + + def forward(self, x, c, feat_rope=None, grid_height=None, grid_width=None): + # Apply activation + c = self.act(c) + + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(6, dim=-1) + + # Attention block + norm_x = self.norm1(x) + modulated_x = modulate(norm_x, shift_msa, scale_msa) + attn_out = self.attn(modulated_x, rope=feat_rope, grid_height=grid_height, grid_width=grid_width) + x = x + gate_msa.unsqueeze(1) * attn_out + + # MLP block + norm_x = self.norm2(x) + modulated_x = modulate(norm_x, shift_mlp, scale_mlp) + mlp_out = self.mlp(modulated_x) + x = x + gate_mlp.unsqueeze(1) * mlp_out + + return x + + +def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0): + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) + grid = np.stack(grid, axis=0) + grid = grid.reshape([2, 1, grid_size, grid_size]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token and extra_tokens > 0: + pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + if embed_dim % 2 != 0: + raise ValueError(f"embed_dim must be divisible by 2, but got {embed_dim}") + + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) + emb = np.concatenate([emb_h, emb_w], axis=1) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + if embed_dim % 2 != 0: + raise ValueError(f"embed_dim must be divisible by 2, but got {embed_dim}") + + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega + + pos = pos.reshape(-1) + out = np.einsum("m,d->md", pos, omega) + + emb_sin = np.sin(out) + emb_cos = np.cos(out) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) + return emb + + +class JiTTransformer2DModel(ModelMixin, ConfigMixin): + r""" + A 2D Transformer for pixel-space class-conditional generation with JiT + ([Back to Basics: Let Denoising Generative Models Denoise](https://arxiv.org/abs/2511.13720)). + + Parameters: + sample_size (`int`, defaults to `256`): + Input image resolution (height and width). + patch_size (`int`, defaults to `16`): + Patch size for the bottleneck patch embedder. + in_channels (`int`, defaults to `3`): + Number of input image channels. + hidden_size (`int`, defaults to `768`): + Transformer hidden dimension. + num_layers (`int`, defaults to `12`): + Number of JiT transformer blocks. + num_attention_heads (`int`, defaults to `12`): + Number of attention heads per block. + mlp_ratio (`float`, defaults to `4.0`): + MLP hidden dimension multiplier. + attention_dropout (`float`, defaults to `0.0`): + Attention dropout in the middle quarter of blocks. + dropout (`float`, defaults to `0.0`): + Projection dropout in the middle quarter of blocks. + num_classes (`int`, defaults to `1000`): + Number of class labels (null label uses index `num_classes` for CFG). + bottleneck_dim (`int`, defaults to `128`): + PCA bottleneck dimension in the patch embedder. + in_context_len (`int`, defaults to `32`): + Number of in-context class tokens prepended mid-network. + in_context_start (`int`, defaults to `4`): + Block index at which in-context tokens are inserted. + norm_eps (`float`, defaults to `1e-6`): + Epsilon for RMSNorm layers. + """ + + _supports_gradient_checkpointing = True + _skip_layerwise_casting_patterns = ["pos_embed", "norm"] + + @register_to_config + def __init__( + self, + sample_size: int = 256, + patch_size: int = 16, + in_channels: int = 3, + hidden_size: int = 768, + num_layers: int = 12, + num_attention_heads: int = 12, + mlp_ratio: float = 4.0, + attention_dropout: float = 0.0, + dropout: float = 0.0, + num_classes: int = 1000, + bottleneck_dim: int = 128, + in_context_len: int = 32, + in_context_start: int = 4, + norm_eps: float = 1e-6, + ): + super().__init__() + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.num_layers = num_layers + self.num_attention_heads = num_attention_heads + self.in_context_len = in_context_len + self.in_context_start = in_context_start + self.norm_eps = norm_eps + self.gradient_checkpointing = False + + # Time and Class Embedding + self.t_embedder = JiTTimestepEmbedder(hidden_size) + self.y_embedder = JiTLabelEmbedder(num_classes, hidden_size) + + # Patch Embedding + self.x_embedder = JiTPatchEmbed( + img_size=sample_size, + patch_size=patch_size, + in_chans=in_channels, + pca_dim=bottleneck_dim, + embed_dim=hidden_size, + bias=True, + ) + + # Positional Embedding (Fixed Sin-Cos) + num_patches = self.x_embedder.num_patches + pos_embed = get_2d_sincos_pos_embed(hidden_size, int(num_patches**0.5)) + self.register_buffer("pos_embed", torch.from_numpy(pos_embed).float().unsqueeze(0), persistent=True) + + # In-context Embedding + if self.in_context_len > 0: + self.in_context_posemb = nn.Parameter(torch.zeros(1, self.in_context_len, hidden_size)) + + # RoPE + half_head_dim = hidden_size // num_attention_heads // 2 + hw_seq_len = sample_size // patch_size + self.feat_rope = JiTRotaryEmbedding(dim=half_head_dim, pt_seq_len=hw_seq_len, num_cls_token=0) + self.feat_rope_incontext = JiTRotaryEmbedding( + dim=half_head_dim, pt_seq_len=hw_seq_len, num_cls_token=self.in_context_len + ) + + # Blocks + self.blocks = nn.ModuleList( + [ + JiTBlock( + hidden_size, + num_attention_heads, + mlp_ratio=mlp_ratio, + attn_drop=attention_dropout if (num_layers // 4 * 3 > i >= num_layers // 4) else 0.0, + proj_drop=dropout if (num_layers // 4 * 3 > i >= num_layers // 4) else 0.0, + eps=norm_eps, + ) + for i in range(num_layers) + ] + ) + + # Final Layer + self.norm_final = RMSNorm(hidden_size, eps=norm_eps) + self.linear_final = nn.Linear(hidden_size, patch_size * patch_size * self.out_channels, bias=True) + self.act_final = nn.SiLU() + self.adaLN_modulation_final = nn.Linear(hidden_size, 2 * hidden_size, bias=True) + + def _get_patch_grid(self, hidden_states): + height, width = hidden_states.shape[-2:] + if height % self.patch_size != 0 or width % self.patch_size != 0: + raise ValueError( + f"Input size {(height, width)} must be divisible by patch_size={self.patch_size}." + ) + return height // self.patch_size, width // self.patch_size + + def _interpolate_pos_encoding(self, tokens, grid_height, grid_width): + num_tokens = grid_height * grid_width + if self.pos_embed.shape[1] == num_tokens: + return self.pos_embed.to(device=tokens.device, dtype=tokens.dtype) + base_size = int(self.pos_embed.shape[1] ** 0.5) + pos_embed = self.pos_embed.reshape(1, base_size, base_size, self.hidden_size).permute(0, 3, 1, 2) + pos_embed = F.interpolate(pos_embed, size=(grid_height, grid_width), mode="bicubic", align_corners=False) + pos_embed = pos_embed.permute(0, 2, 3, 1).reshape(1, num_tokens, self.hidden_size) + return pos_embed.to(device=tokens.device, dtype=tokens.dtype) + + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.LongTensor, + class_labels: torch.LongTensor, + interpolate_pos_encoding: bool = True, + return_dict: bool = True, + ): + + t_emb = self.t_embedder(timestep, dtype=hidden_states.dtype) + y_emb = self.y_embedder(class_labels) + + # Ensure embeddings match hidden_states dtype + y_emb = y_emb.to(dtype=hidden_states.dtype) + + c = t_emb + y_emb + + # Patch Embed + grid_height, grid_width = self._get_patch_grid(hidden_states) + x = self.x_embedder(hidden_states) + if interpolate_pos_encoding: + pos_embed = self._interpolate_pos_encoding(x, grid_height, grid_width) + else: + expected_tokens = grid_height * grid_width + if self.pos_embed.shape[1] != expected_tokens: + raise ValueError( + f"pos_embed token count {self.pos_embed.shape[1]} does not match input token count {expected_tokens}. " + "Enable interpolate_pos_encoding for dynamic resolutions." + ) + pos_embed = self.pos_embed.to(device=x.device, dtype=x.dtype) + x = x + pos_embed + + # Blocks + for i, block in enumerate(self.blocks): + if self.in_context_len > 0 and i == self.in_context_start: + in_context_tokens = y_emb.unsqueeze(1).repeat(1, self.in_context_len, 1) + in_context_tokens = in_context_tokens + self.in_context_posemb.to(in_context_tokens.dtype) + x = torch.cat([in_context_tokens, x], dim=1) + + rope = self.feat_rope if i < self.in_context_start else self.feat_rope_incontext + + if self.training and self.gradient_checkpointing: + def custom_forward(current_x, current_c): + return block( + current_x, + current_c, + feat_rope=rope, + grid_height=grid_height, + grid_width=grid_width, + ) + + x = torch.utils.checkpoint.checkpoint( + custom_forward, + x, + c, + use_reentrant=False, + ) + else: + x = block(x, c, feat_rope=rope, grid_height=grid_height, grid_width=grid_width) + + # Slice off in-context tokens + if self.in_context_len > 0: + x = x[:, self.in_context_len :] + + # Final Layer + c = self.act_final(c) + shift, scale = self.adaLN_modulation_final(c).chunk(2, dim=1) + + x = modulate(self.norm_final(x), shift, scale) + x = self.linear_final(x) + + # Unpatchify + x = x.reshape(shape=(x.shape[0], grid_height, grid_width, self.patch_size, self.patch_size, self.out_channels)) + x = torch.einsum("nhwpqc->nchpwq", x) + output = x.reshape( + shape=(x.shape[0], self.out_channels, grid_height * self.patch_size, grid_width * self.patch_size) + ) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) diff --git a/JiT-L-16-SIM/model_index.json b/JiT-L-16-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..6ae22d786e6a73f7b2c61a207ad9634f9a2fef4e --- /dev/null +++ b/JiT-L-16-SIM/model_index.json @@ -0,0 +1,1017 @@ +{ + "_class_name": [ + "pipeline", + "JiTPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_flow_match_fd", + "FDLossFlowMatchScheduler" + ], + "transformer": [ + "jit_transformer_2d", + "JiTTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + } +} diff --git a/JiT-L-16-SIM/pipeline.py b/JiT-L-16-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..164b44e829eeedda7dfae5ee8d456a3252082097 --- /dev/null +++ b/JiT-L-16-SIM/pipeline.py @@ -0,0 +1,350 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hub custom pipeline: JiTPipeline for FD-Loss post-trained JiT (legacy time, velocity Euler).""" + +from __future__ import annotations + +import importlib.util +import inspect +import json +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union + +import torch +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from pathlib import Path + >>> import torch + >>> from diffusers import DiffusionPipeline + + >>> model_dir = Path("BiliSakura/FD-Loss-diffusers/JiT-B-16-SIM") + >>> pipe = DiffusionPipeline.from_pretrained( + ... str(model_dir), + ... custom_pipeline=str(model_dir / "pipeline.py"), + ... trust_remote_code=True, + ... ) + >>> pipe.to("cuda") + >>> image = pipe(class_labels="golden retriever", num_inference_steps=1, guidance_scale=3.0).images[0] + ``` +""" + +RECOMMENDED_NOISE_BY_SIZE = {256: 1.0, 512: 2.0} + + +class JiTPipeline(DiffusionPipeline): + r""" + Class-conditional JiT pipeline for FD-Loss checkpoints (1-NFE velocity Euler, legacy time convention). + + Args: + transformer ([`JiTTransformer2DModel`]): JiT backbone. + scheduler ([`FDLossFlowMatchScheduler`], *optional*): Bundled flow-matching scheduler (`t=1→0`). + Accepts `generator` in `step()` for reproducible stochastic sampling. + id2label (`dict[int, str]`, *optional*): ImageNet id → label strings from `model_index.json`. + """ + + model_cpu_offload_seq = "transformer" + _optional_components = ["scheduler"] + + @staticmethod + def prepare_extra_step_kwargs( + scheduler, + generator=None, + eta: float | None = None, + ): + kwargs = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + @staticmethod + def _coerce_scheduler(scheduler, transformer): + """Always load the bundled FD-Loss flow scheduler (ignore model_index diffusers schedulers).""" + variant_path = getattr(transformer.config, "_name_or_path", None) + if variant_path: + scheduler_dir = Path(variant_path).resolve().parent / "scheduler" + module_path = scheduler_dir / "scheduling_flow_match_fd.py" + config_path = scheduler_dir / "scheduler_config.json" + if module_path.is_file() and config_path.is_file(): + spec = importlib.util.spec_from_file_location("scheduling_flow_match_fd", module_path) + if spec is not None and spec.loader is not None: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + scheduler_cls = getattr(module, "FDLossFlowMatchScheduler") + return scheduler_cls.from_pretrained(str(scheduler_dir)) + scheduler_dir = Path(__file__).resolve().parent / "scheduler" + module_path = scheduler_dir / "scheduling_flow_match_fd.py" + config_path = scheduler_dir / "scheduler_config.json" + if module_path.is_file() and config_path.is_file(): + spec = importlib.util.spec_from_file_location("scheduling_flow_match_fd", module_path) + if spec is not None and spec.loader is not None: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + scheduler_cls = getattr(module, "FDLossFlowMatchScheduler") + return scheduler_cls.from_pretrained(str(scheduler_dir)) + raise ValueError("FD-Loss JiT requires a bundled FDLossFlowMatchScheduler under `scheduler/`.") + + def __init__( + self, + transformer, + scheduler=None, + id2label: Optional[Dict[Union[int, str], str]] = None, + ): + super().__init__() + self.legacy_time_convention = True + scheduler = self._coerce_scheduler(scheduler, transformer) + self.register_modules(transformer=transformer, scheduler=scheduler) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + r"""Map ImageNet English label strings to class ids.""" + self._ensure_labels_loaded() + if isinstance(label, str): + label = [label] + missing = [item for item in label if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Examples: {preview}, ...") + return [self.labels[item] for item in label] + + def _backbone_timestep(self, t: torch.Tensor) -> torch.Tensor: + if self.legacy_time_convention: + return 1.0 - t + return t + + def _cfg_interval(self, interval_min: float, interval_max: float, device: torch.device) -> Tuple[float, float]: + low = float(self._backbone_timestep(torch.tensor(interval_min, device=device))) + high = float(self._backbone_timestep(torch.tensor(interval_max, device=device))) + return min(low, high), max(low, high) + + def _flow_time_tensors(self, t: torch.Tensor, sample: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Broadcast scalar flow time and per-sample backbone timesteps for `sample` (B, C, H, W).""" + flow_t = t.reshape(()).to(device=sample.device, dtype=sample.dtype) + t_view = flow_t.reshape(*([1] * sample.ndim)) + t_bb = self._backbone_timestep(flow_t).expand(sample.shape[0]) + return t_view, t_bb + + def _predict_velocity( + self, + sample: torch.Tensor, + t: torch.Tensor, + class_labels: torch.Tensor, + guidance_scale: float, + cfg_interval: Tuple[float, float], + t_eps: float, + interpolate_pos_encoding: bool, + ) -> torch.Tensor: + t_view, t_bb = self._flow_time_tensors(t, sample) + num_classes = int(getattr(self.transformer.config, "num_classes", 1000)) + + x_cond = self.transformer( + sample, + timestep=t_bb, + class_labels=class_labels, + interpolate_pos_encoding=interpolate_pos_encoding, + ).sample + v_cond = (sample - x_cond) / t_view.clamp_min(t_eps) + + if guidance_scale <= 1.0: + return v_cond + + class_null = torch.full_like(class_labels, num_classes) + x_uncond = self.transformer( + sample, + timestep=t_bb, + class_labels=class_null, + interpolate_pos_encoding=interpolate_pos_encoding, + ).sample + v_uncond = (sample - x_uncond) / t_view.clamp_min(t_eps) + + low, high = cfg_interval + flow_t = t.reshape(()).to(device=sample.device, dtype=sample.dtype) + mask = (flow_t < high) & ((low == 0) | (flow_t > low)) + scale = torch.where( + mask, + torch.tensor(guidance_scale, device=sample.device, dtype=sample.dtype), + torch.tensor(1.0, device=sample.device, dtype=sample.dtype), + ) + while scale.ndim < v_cond.ndim: + scale = scale.unsqueeze(-1) + return v_uncond + scale * (v_cond - v_uncond) + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + path = getattr(self.config, "_name_or_path", None) + if path: + model_index_path = Path(path).resolve() / "model_index.json" + if model_index_path.exists(): + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if isinstance(id2label, dict): + self._id2label = {int(k): v for k, v in id2label.items()} + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) + return list(class_labels) + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + guidance_scale: float = 3.0, + guidance_interval_min: float = 0.1, + guidance_interval_max: float = 1.0, + noise_scale: Optional[float] = None, + t_eps: float = 5e-2, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + num_inference_steps: int = 1, + height: Optional[int] = None, + width: Optional[int] = None, + interpolate_pos_encoding: bool = True, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + r""" + Generate images conditioned on ImageNet class ids or English labels. + + Args: + class_labels: Class id(s), label string(s), or mixed list. + guidance_scale: CFG scale (FD-Loss JiT-B default: 3.0). + guidance_interval_min: Lower flow-time bound for CFG (default 0.1). + guidance_interval_max: Upper flow-time bound for CFG (default 1.0). + noise_scale: Initial Gaussian scale (default 1.0 at 256px). + t_eps: Clamp for flow time in velocity denominator. + generator: RNG for latent init. + num_inference_steps: FD-Loss sampling steps (1 = single NFE). + height: Output height (defaults to native `sample_size`). + width: Output width (defaults to native `sample_size`). + interpolate_pos_encoding: Interpolate positional embeddings for non-native sizes. + output_type: `pil`, `np`, or `pt`. + return_dict: Return [`ImagePipelineOutput`] if True. + """ + # 1. Check inputs + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + if output_type not in {"pil", "np", "pt"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt'.") + + # 2. Call parameters + class_label_ids = self._normalize_class_labels(class_labels) + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + image_size = int(self.transformer.config.sample_size) + patch_size = int(self.transformer.config.patch_size) + height = int(height or image_size) + width = int(width or image_size) + if height % patch_size != 0 or width % patch_size != 0: + raise ValueError(f"height/width must be divisible by patch_size={patch_size}.") + channels = int(self.transformer.config.in_channels) + null_class_val = int(getattr(self.transformer.config, "num_classes", 1000)) + if noise_scale is None: + noise_scale = RECOMMENDED_NOISE_BY_SIZE.get(max(height, width), 1.0) + + # 3. Class labels + labels = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1) + labels = labels.clamp(0, null_class_val - 1) + cfg_interval = self._cfg_interval(guidance_interval_min, guidance_interval_max, labels.device) + + # 4. Latents + latents = randn_tensor( + (batch_size, channels, height, width), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) * noise_scale + + # 5–6. Denoising loop (FD-Loss flow t=1 → 0; generator forwarded to scheduler.step) + self.scheduler.set_timesteps(num_inference_steps, device=self._execution_device) + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + timesteps = self.scheduler.timesteps + + for i in self.progress_bar(range(num_inference_steps)): + t_cur = timesteps[i] + velocity = self._predict_velocity( + latents, t_cur, labels, guidance_scale, cfg_interval, t_eps, interpolate_pos_encoding + ) + latents = self.scheduler.step(velocity, t_cur, latents, **extra_step_kwargs).prev_sample + + # 8. Output + images_pt = ((latents.float().clamp(-1, 1) + 1.0) / 2.0).cpu() + if output_type == "pt": + images = images_pt + elif output_type == "np": + images = images_pt.permute(0, 2, 3, 1).numpy() + else: + images = self.numpy_to_pil(images_pt.permute(0, 2, 3, 1).numpy()) + + self.maybe_free_model_hooks() + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) diff --git a/JiT-L-16-SIM/scheduler/scheduler_config.json b/JiT-L-16-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..fda98c585a0e298556a0a75e5f051b6d915ff9e2 --- /dev/null +++ b/JiT-L-16-SIM/scheduler/scheduler_config.json @@ -0,0 +1,6 @@ +{ + "_class_name": "FDLossFlowMatchScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000, + "stochastic_sampling": false +} diff --git a/JiT-L-16-SIM/scheduler/scheduling_flow_match_fd.py b/JiT-L-16-SIM/scheduler/scheduling_flow_match_fd.py new file mode 100644 index 0000000000000000000000000000000000000000..21872a41d87c934e4dd396e33887b19c1d825382 --- /dev/null +++ b/JiT-L-16-SIM/scheduler/scheduling_flow_match_fd.py @@ -0,0 +1,110 @@ +# Copyright 2026 The FD-Loss Authors. SPDX-License-Identifier: MIT +"""Flow-matching scheduler for FD-Loss JiT sampling (t: 1 -> 0).""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class FDLossFlowMatchSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class FDLossFlowMatchScheduler(SchedulerMixin, ConfigMixin): + """Euler flow-matching scheduler with optional stochastic steps and generator-aware noise.""" + + order = 1 + config_name = "scheduler_config.json" + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + stochastic_sampling: bool = False, + ): + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> torch.Tensor: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + return self.timesteps + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.reshape(-1)[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + @staticmethod + def _expand_time(value: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + value = value.to(dtype=sample.dtype, device=sample.device) + while value.ndim < sample.ndim: + value = value.unsqueeze(-1) + return value + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[FDLossFlowMatchSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = self._expand_time(t - t_next, sample) + + if self.config.stochastic_sampling: + t_expanded = self._expand_time(t, sample) + t_next_expanded = self._expand_time(t_next, sample) + x0 = sample - t_expanded * model_output + noise = randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + prev_sample = (1.0 - t_next_expanded) * x0 + t_next_expanded * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return FDLossFlowMatchSchedulerOutput(prev_sample=prev_sample) diff --git a/JiT-L-16-SIM/transformer/config.json b/JiT-L-16-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e0d13b6090911eeafaa0378a92f8f8615740369 --- /dev/null +++ b/JiT-L-16-SIM/transformer/config.json @@ -0,0 +1,18 @@ +{ + "_class_name": "JiTTransformer2DModel", + "_diffusers_version": "0.36.0", + "attention_dropout": 0.0, + "bottleneck_dim": 128, + "dropout": 0.0, + "hidden_size": 1024, + "in_channels": 3, + "in_context_len": 32, + "in_context_start": 8, + "mlp_ratio": 4.0, + "norm_eps": 1e-06, + "num_attention_heads": 16, + "num_classes": 1000, + "num_layers": 24, + "patch_size": 16, + "sample_size": 256 +} diff --git a/JiT-L-16-SIM/transformer/diffusion_pytorch_model.safetensors b/JiT-L-16-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..de36a71e3a10d6a4123ed4d7503b5b8e215b532f --- /dev/null +++ b/JiT-L-16-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06c08d90b58a5423cc1db3cb5cd3537f6f42349e70abadca08ea48135a76d183 +size 1836593656 diff --git a/JiT-L-16-SIM/transformer/jit_transformer_2d.py b/JiT-L-16-SIM/transformer/jit_transformer_2d.py new file mode 100644 index 0000000000000000000000000000000000000000..1f14b1fc440e7c3730c295f0a90b883e32c12322 --- /dev/null +++ b/JiT-L-16-SIM/transformer/jit_transformer_2d.py @@ -0,0 +1,559 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm +from diffusers.utils import logging + +logger = logging.get_logger(__name__) + + + +def broadcat(tensors, dim=-1): + num_tensors = len(tensors) + shape_lens = {len(t.shape) for t in tensors} + if len(shape_lens) != 1: + raise ValueError("tensors must all have the same number of dimensions") + shape_len = list(shape_lens)[0] + dim = (dim + shape_len) if dim < 0 else dim + dims = list(zip(*(list(t.shape) for t in tensors))) + expandable_dims = [(i, val) for i, val in enumerate(dims) if i != dim] + + if not all(len(set(t[1])) <= 2 for t in expandable_dims): + raise ValueError("invalid dimensions for broadcastable concatenation") + + max_dims = [(t[0], max(t[1])) for t in expandable_dims] + expanded_dims = [(t[0], (t[1],) * num_tensors) for t in max_dims] + expanded_dims.insert(dim, (dim, dims[dim])) + expandable_shapes = list(zip(*(t[1] for t in expanded_dims))) + tensors = [t[0].expand(*t[1]) for t in zip(tensors, expandable_shapes)] + return torch.cat(tensors, dim=dim) + + +def rotate_half(x): + x = x.view(*x.shape[:-1], x.shape[-1] // 2, 2) + x1, x2 = x.unbind(dim=-1) + x = torch.stack((-x2, x1), dim=-1) + return x.view(*x.shape[:-2], -1) + + +class JiTRotaryEmbedding(nn.Module): + def __init__( + self, + dim, + pt_seq_len=16, + ft_seq_len=None, + custom_freqs=None, + theta=10000, + num_cls_token=0, + ): + super().__init__() + self.dim = dim + self.pt_seq_len = pt_seq_len + self.theta = theta + self.num_cls_token = num_cls_token + self.custom_freqs = custom_freqs + if ft_seq_len is None: + ft_seq_len = pt_seq_len + self._cached_hw = None + cos, sin = self._build_freqs(ft_seq_len, ft_seq_len, device=torch.device("cpu")) + self.register_buffer("freqs_cos", cos, persistent=False) + self.register_buffer("freqs_sin", sin, persistent=False) + self._cached_hw = (ft_seq_len, ft_seq_len) + + def _build_freqs(self, height, width, device): + if self.custom_freqs is not None: + freqs = self.custom_freqs.to(device=device, dtype=torch.float32) + else: + freqs = 1.0 / ( + self.theta ** (torch.arange(0, self.dim, 2, device=device, dtype=torch.float32)[: (self.dim // 2)] / self.dim) + ) + + t_h = torch.arange(height, device=device, dtype=torch.float32) / height * self.pt_seq_len + t_w = torch.arange(width, device=device, dtype=torch.float32) / width * self.pt_seq_len + freqs_h = torch.einsum("..., f -> ... f", t_h, freqs).repeat_interleave(2, dim=-1) + freqs_w = torch.einsum("..., f -> ... f", t_w, freqs).repeat_interleave(2, dim=-1) + freqs_2d = broadcat((freqs_h[:, None, :], freqs_w[None, :, :]), dim=-1) + freqs_flat = freqs_2d.view(-1, freqs_2d.shape[-1]) + cos_img = freqs_flat.cos() + sin_img = freqs_flat.sin() + if self.num_cls_token > 0: + _, dim_freq = cos_img.shape + cos_pad = torch.ones(self.num_cls_token, dim_freq, dtype=cos_img.dtype, device=device) + sin_pad = torch.zeros(self.num_cls_token, dim_freq, dtype=sin_img.dtype, device=device) + cos_img = torch.cat([cos_pad, cos_img], dim=0) + sin_img = torch.cat([sin_pad, sin_img], dim=0) + return cos_img, sin_img + + def forward(self, t, height=None, width=None): + # Applied on (batch, seq_len, heads, head_dim) tensors from attention. + seq_len = t.shape[1] + if height is None or width is None: + image_tokens = seq_len - self.num_cls_token + size = int(image_tokens**0.5) + if size * size != image_tokens: + raise ValueError( + f"Cannot infer square token grid from sequence length {seq_len} with {self.num_cls_token} class tokens." + ) + height = size + width = size + if self._cached_hw != (height, width) or self.freqs_cos.device != t.device: + self.freqs_cos, self.freqs_sin = self._build_freqs(height, width, device=t.device) + self._cached_hw = (height, width) + freqs_cos = self.freqs_cos[:seq_len].to(t.dtype) + freqs_sin = self.freqs_sin[:seq_len].to(t.dtype) + + return t * freqs_cos[:, None, :] + rotate_half(t) * freqs_sin[:, None, :] + + +def modulate(x, shift, scale): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +class JiTPatchEmbed(nn.Module): + """Image to Patch Embedding with Bottleneck""" + + def __init__(self, img_size=224, patch_size=16, in_chans=3, pca_dim=768, embed_dim=768, bias=True): + super().__init__() + img_size = (img_size, img_size) + patch_size = (patch_size, patch_size) + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0]) + + self.proj1 = nn.Conv2d(in_chans, pca_dim, kernel_size=patch_size, stride=patch_size, bias=False) + self.proj2 = nn.Conv2d(pca_dim, embed_dim, kernel_size=1, stride=1, bias=bias) + + def forward(self, x): + x = self.proj2(self.proj1(x)).flatten(2).transpose(1, 2) + return x + + +class JiTTimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + + def __init__(self, hidden_size, frequency_embedding_size=256): + super().__init__() + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=True), + nn.SiLU(), + nn.Linear(hidden_size, hidden_size, bias=True), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t, dim, max_period=10000): + """ + Create sinusoidal timestep embeddings. + """ + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half + ).to(device=t.device) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t, dtype=None): + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + if dtype is not None: + t_freq = t_freq.to(dtype=dtype) + t_emb = self.mlp(t_freq) + return t_emb + + +class JiTLabelEmbedder(nn.Module): + """ + Embeds class labels into vector representations. + """ + + def __init__(self, num_classes, hidden_size): + super().__init__() + self.embedding_table = nn.Embedding(num_classes + 1, hidden_size) + self.num_classes = num_classes + + def forward(self, labels): + embeddings = self.embedding_table(labels) + return embeddings + + +class JiTAttention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=True, qk_norm=True, attn_drop=0.0, proj_drop=0.0, eps=1e-6): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + + self.q_norm = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity() + self.k_norm = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity() + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = attn_drop + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, rope=None, grid_height=None, grid_width=None): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + + q = self.q_norm(q) + k = self.k_norm(k) + + if rope is not None: + q = q.transpose(1, 2) + k = k.transpose(1, 2) + q = rope(q, height=grid_height, width=grid_width) + k = rope(k, height=grid_height, width=grid_width) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + + dropout_p = self.attn_drop if self.training else 0.0 + x = F.scaled_dot_product_attention(q, k, v, dropout_p=dropout_p) + x = x.transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class JiTSwiGLUFFN(nn.Module): + def __init__(self, dim: int, hidden_dim: int, drop=0.0, bias=True) -> None: + super().__init__() + hidden_dim = int(hidden_dim * 2 / 3) + self.w12 = nn.Linear(dim, 2 * hidden_dim, bias=bias) + self.w3 = nn.Linear(hidden_dim, dim, bias=bias) + self.ffn_dropout = nn.Dropout(drop) + + def forward(self, x): + x12 = self.w12(x) + x1, x2 = x12.chunk(2, dim=-1) + hidden = F.silu(x1) * x2 + return self.w3(self.ffn_dropout(hidden)) + + +class JiTBlock(nn.Module): + def __init__(self, hidden_size, num_heads, mlp_ratio=4.0, attn_drop=0.0, proj_drop=0.0, eps=1e-6): + super().__init__() + self.norm1 = RMSNorm(hidden_size, eps=eps) + self.attn = JiTAttention( + hidden_size, + num_heads=num_heads, + qkv_bias=True, + qk_norm=True, + attn_drop=attn_drop, + proj_drop=proj_drop, + eps=eps, + ) + self.norm2 = RMSNorm(hidden_size, eps=eps) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + self.mlp = JiTSwiGLUFFN(hidden_size, mlp_hidden_dim, drop=proj_drop) + + self.act = nn.SiLU() + self.adaLN_modulation = nn.Linear(hidden_size, 6 * hidden_size, bias=True) + + def forward(self, x, c, feat_rope=None, grid_height=None, grid_width=None): + # Apply activation + c = self.act(c) + + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(6, dim=-1) + + # Attention block + norm_x = self.norm1(x) + modulated_x = modulate(norm_x, shift_msa, scale_msa) + attn_out = self.attn(modulated_x, rope=feat_rope, grid_height=grid_height, grid_width=grid_width) + x = x + gate_msa.unsqueeze(1) * attn_out + + # MLP block + norm_x = self.norm2(x) + modulated_x = modulate(norm_x, shift_mlp, scale_mlp) + mlp_out = self.mlp(modulated_x) + x = x + gate_mlp.unsqueeze(1) * mlp_out + + return x + + +def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0): + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) + grid = np.stack(grid, axis=0) + grid = grid.reshape([2, 1, grid_size, grid_size]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token and extra_tokens > 0: + pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + if embed_dim % 2 != 0: + raise ValueError(f"embed_dim must be divisible by 2, but got {embed_dim}") + + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) + emb = np.concatenate([emb_h, emb_w], axis=1) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + if embed_dim % 2 != 0: + raise ValueError(f"embed_dim must be divisible by 2, but got {embed_dim}") + + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega + + pos = pos.reshape(-1) + out = np.einsum("m,d->md", pos, omega) + + emb_sin = np.sin(out) + emb_cos = np.cos(out) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) + return emb + + +class JiTTransformer2DModel(ModelMixin, ConfigMixin): + r""" + A 2D Transformer for pixel-space class-conditional generation with JiT + ([Back to Basics: Let Denoising Generative Models Denoise](https://arxiv.org/abs/2511.13720)). + + Parameters: + sample_size (`int`, defaults to `256`): + Input image resolution (height and width). + patch_size (`int`, defaults to `16`): + Patch size for the bottleneck patch embedder. + in_channels (`int`, defaults to `3`): + Number of input image channels. + hidden_size (`int`, defaults to `768`): + Transformer hidden dimension. + num_layers (`int`, defaults to `12`): + Number of JiT transformer blocks. + num_attention_heads (`int`, defaults to `12`): + Number of attention heads per block. + mlp_ratio (`float`, defaults to `4.0`): + MLP hidden dimension multiplier. + attention_dropout (`float`, defaults to `0.0`): + Attention dropout in the middle quarter of blocks. + dropout (`float`, defaults to `0.0`): + Projection dropout in the middle quarter of blocks. + num_classes (`int`, defaults to `1000`): + Number of class labels (null label uses index `num_classes` for CFG). + bottleneck_dim (`int`, defaults to `128`): + PCA bottleneck dimension in the patch embedder. + in_context_len (`int`, defaults to `32`): + Number of in-context class tokens prepended mid-network. + in_context_start (`int`, defaults to `4`): + Block index at which in-context tokens are inserted. + norm_eps (`float`, defaults to `1e-6`): + Epsilon for RMSNorm layers. + """ + + _supports_gradient_checkpointing = True + _skip_layerwise_casting_patterns = ["pos_embed", "norm"] + + @register_to_config + def __init__( + self, + sample_size: int = 256, + patch_size: int = 16, + in_channels: int = 3, + hidden_size: int = 768, + num_layers: int = 12, + num_attention_heads: int = 12, + mlp_ratio: float = 4.0, + attention_dropout: float = 0.0, + dropout: float = 0.0, + num_classes: int = 1000, + bottleneck_dim: int = 128, + in_context_len: int = 32, + in_context_start: int = 4, + norm_eps: float = 1e-6, + ): + super().__init__() + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.num_layers = num_layers + self.num_attention_heads = num_attention_heads + self.in_context_len = in_context_len + self.in_context_start = in_context_start + self.norm_eps = norm_eps + self.gradient_checkpointing = False + + # Time and Class Embedding + self.t_embedder = JiTTimestepEmbedder(hidden_size) + self.y_embedder = JiTLabelEmbedder(num_classes, hidden_size) + + # Patch Embedding + self.x_embedder = JiTPatchEmbed( + img_size=sample_size, + patch_size=patch_size, + in_chans=in_channels, + pca_dim=bottleneck_dim, + embed_dim=hidden_size, + bias=True, + ) + + # Positional Embedding (Fixed Sin-Cos) + num_patches = self.x_embedder.num_patches + pos_embed = get_2d_sincos_pos_embed(hidden_size, int(num_patches**0.5)) + self.register_buffer("pos_embed", torch.from_numpy(pos_embed).float().unsqueeze(0), persistent=True) + + # In-context Embedding + if self.in_context_len > 0: + self.in_context_posemb = nn.Parameter(torch.zeros(1, self.in_context_len, hidden_size)) + + # RoPE + half_head_dim = hidden_size // num_attention_heads // 2 + hw_seq_len = sample_size // patch_size + self.feat_rope = JiTRotaryEmbedding(dim=half_head_dim, pt_seq_len=hw_seq_len, num_cls_token=0) + self.feat_rope_incontext = JiTRotaryEmbedding( + dim=half_head_dim, pt_seq_len=hw_seq_len, num_cls_token=self.in_context_len + ) + + # Blocks + self.blocks = nn.ModuleList( + [ + JiTBlock( + hidden_size, + num_attention_heads, + mlp_ratio=mlp_ratio, + attn_drop=attention_dropout if (num_layers // 4 * 3 > i >= num_layers // 4) else 0.0, + proj_drop=dropout if (num_layers // 4 * 3 > i >= num_layers // 4) else 0.0, + eps=norm_eps, + ) + for i in range(num_layers) + ] + ) + + # Final Layer + self.norm_final = RMSNorm(hidden_size, eps=norm_eps) + self.linear_final = nn.Linear(hidden_size, patch_size * patch_size * self.out_channels, bias=True) + self.act_final = nn.SiLU() + self.adaLN_modulation_final = nn.Linear(hidden_size, 2 * hidden_size, bias=True) + + def _get_patch_grid(self, hidden_states): + height, width = hidden_states.shape[-2:] + if height % self.patch_size != 0 or width % self.patch_size != 0: + raise ValueError( + f"Input size {(height, width)} must be divisible by patch_size={self.patch_size}." + ) + return height // self.patch_size, width // self.patch_size + + def _interpolate_pos_encoding(self, tokens, grid_height, grid_width): + num_tokens = grid_height * grid_width + if self.pos_embed.shape[1] == num_tokens: + return self.pos_embed.to(device=tokens.device, dtype=tokens.dtype) + base_size = int(self.pos_embed.shape[1] ** 0.5) + pos_embed = self.pos_embed.reshape(1, base_size, base_size, self.hidden_size).permute(0, 3, 1, 2) + pos_embed = F.interpolate(pos_embed, size=(grid_height, grid_width), mode="bicubic", align_corners=False) + pos_embed = pos_embed.permute(0, 2, 3, 1).reshape(1, num_tokens, self.hidden_size) + return pos_embed.to(device=tokens.device, dtype=tokens.dtype) + + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.LongTensor, + class_labels: torch.LongTensor, + interpolate_pos_encoding: bool = True, + return_dict: bool = True, + ): + + t_emb = self.t_embedder(timestep, dtype=hidden_states.dtype) + y_emb = self.y_embedder(class_labels) + + # Ensure embeddings match hidden_states dtype + y_emb = y_emb.to(dtype=hidden_states.dtype) + + c = t_emb + y_emb + + # Patch Embed + grid_height, grid_width = self._get_patch_grid(hidden_states) + x = self.x_embedder(hidden_states) + if interpolate_pos_encoding: + pos_embed = self._interpolate_pos_encoding(x, grid_height, grid_width) + else: + expected_tokens = grid_height * grid_width + if self.pos_embed.shape[1] != expected_tokens: + raise ValueError( + f"pos_embed token count {self.pos_embed.shape[1]} does not match input token count {expected_tokens}. " + "Enable interpolate_pos_encoding for dynamic resolutions." + ) + pos_embed = self.pos_embed.to(device=x.device, dtype=x.dtype) + x = x + pos_embed + + # Blocks + for i, block in enumerate(self.blocks): + if self.in_context_len > 0 and i == self.in_context_start: + in_context_tokens = y_emb.unsqueeze(1).repeat(1, self.in_context_len, 1) + in_context_tokens = in_context_tokens + self.in_context_posemb.to(in_context_tokens.dtype) + x = torch.cat([in_context_tokens, x], dim=1) + + rope = self.feat_rope if i < self.in_context_start else self.feat_rope_incontext + + if self.training and self.gradient_checkpointing: + def custom_forward(current_x, current_c): + return block( + current_x, + current_c, + feat_rope=rope, + grid_height=grid_height, + grid_width=grid_width, + ) + + x = torch.utils.checkpoint.checkpoint( + custom_forward, + x, + c, + use_reentrant=False, + ) + else: + x = block(x, c, feat_rope=rope, grid_height=grid_height, grid_width=grid_width) + + # Slice off in-context tokens + if self.in_context_len > 0: + x = x[:, self.in_context_len :] + + # Final Layer + c = self.act_final(c) + shift, scale = self.adaLN_modulation_final(c).chunk(2, dim=1) + + x = modulate(self.norm_final(x), shift, scale) + x = self.linear_final(x) + + # Unpatchify + x = x.reshape(shape=(x.shape[0], grid_height, grid_width, self.patch_size, self.patch_size, self.out_channels)) + x = torch.einsum("nhwpqc->nchpwq", x) + output = x.reshape( + shape=(x.shape[0], self.out_channels, grid_height * self.patch_size, grid_width * self.patch_size) + ) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1169a3f471764acd290cdefa9b718bc50e54defe --- /dev/null +++ b/README.md @@ -0,0 +1,255 @@ +--- +license: mit +library_name: diffusers +pipeline_tag: text-to-image +tags: +- diffusers +- fd-loss +- jit +- imf +- pmf +- image-generation +- class-conditional +- imagenet +inference: true +widget: +- output: + url: JiT-H-16-SIM/demo.png +language: +- en +--- + +# FD-Loss-diffusers + +Post-training checkpoints with the same JiT / iMF / pMF architectures as the base models, distilled with FD-loss (feature distillation). + +- **JiT**: all `/16` variants for 256px generation +- **iMF**: all variants for 256px generation +- **pMF**: both `/16` and `/32` +- **SIM** suffix: SigLIP + Inception + MAE FD-loss +- Default inference: **1 NFE** (`num_inference_steps=1`) + +FD-Loss JiT uses **legacy time convention** and **velocity Euler** via bundled `FDLossFlowMatchScheduler` (`scheduler/scheduling_flow_match_fd.py`; timesteps `t=1→0`). + +## Demo + +![JiT-H-16-SIM demo](JiT-H-16-SIM/demo.png) + +Class-conditional sample (ImageNet class **golden retriever**), `JiT-H/16` FD-SIM at 256×256, 1 NFE, CFG 2.2, interval [0.1, 1.0], seed 42. + +## Inference (`JiT-B-16-SIM`) + +```python +from pathlib import Path +import torch +from diffusers import DiffusionPipeline + +model_dir = Path("./JiT-B-16-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, +) +pipe.to("cuda") + +generator = torch.Generator(device="cuda").manual_seed(42) +image = pipe( + class_labels="golden retriever", + num_inference_steps=1, # 1 NFE (FD-Loss default) + guidance_scale=3.0, + guidance_interval_min=0.1, + guidance_interval_max=1.0, + generator=generator, +).images[0] +``` + +| Parameter | JiT-B-16-SIM default | Source | +| --- | --- | --- | +| `num_inference_steps` | `1` | `--num_sampling_steps 1` | +| `guidance_scale` | `3.0` | JiT_B eval preset | +| `guidance_interval_min` / `max` | `0.1` / `1.0` | JiT_B eval preset | +| `legacy_time_convention` | `True` (pipeline default) | `--legacy_time_convention` | + +## Inference (`JiT-L-16-SIM`) + +Same loading pattern as JiT-B; use **CFG 2.4** (JiT_L eval preset): + +```python +model_dir = Path("./JiT-L-16-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, +).to("cuda") + +image = pipe( + class_labels="golden retriever", + num_inference_steps=1, + guidance_scale=2.4, + generator=torch.Generator(device="cuda").manual_seed(42), +).images[0] +``` + +## Inference (`JiT-H-16-SIM`) + +Same loading pattern as JiT-B/L; use **CFG 2.2** (JiT_H eval preset): + +```python +model_dir = Path("./JiT-H-16-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, +).to("cuda") + +image = pipe( + class_labels="golden retriever", + num_inference_steps=1, + guidance_scale=2.2, + guidance_interval_min=0.1, + guidance_interval_max=1.0, + generator=torch.Generator(device="cuda").manual_seed(42), +).images[0] +``` + + +## Available variants + +| Variant | Path | Architecture | Resolution | CFG (1 NFE) | +| --- | --- | --- | --- | --- | +| JiT-B-16-SIM | `./JiT-B-16-SIM` | JiT-B/16 | 256×256 | 3.0 | +| JiT-L-16-SIM | `./JiT-L-16-SIM` | JiT-L/16 | 256×256 | 2.4 | +| JiT-H-16-SIM | `./JiT-H-16-SIM` | JiT-H/16 | 256×256 | 2.2 | +| iMF-B-SIM | `./iMF-B-SIM` | iMF-B/2 | 256×256 | 8.0 | +| iMF-L-SIM | `./iMF-L-SIM` | iMF-L/2 | 256×256 | 8.0 | +| iMF-XL-SIM | `./iMF-XL-SIM` | iMF-XL/2 | 256×256 | 8.0 | +| pMF-B-16-SIM | `./pMF-B-16-SIM` | pMF-B/16 | 256×256 | 7.5 | +| pMF-B-32-SIM | `./pMF-B-32-SIM` | pMF-B/32 | 512×512 | 6.5 | +| pMF-L-16-SIM | `./pMF-L-16-SIM` | pMF-L/16 | 256×256 | 7.0 | +| pMF-L-32-SIM | `./pMF-L-32-SIM` | pMF-L/32 | 512×512 | 7.5 | +| pMF-H-32-SIM | `./pMF-H-32-SIM` | pMF-H/32 | 512×512 | 5.5 | + +## Inference (`iMF-B-SIM`) + +Uses production `IMFPipeline` from `iMF-diffusers/iMF-B-2` (native iMF time convention, not legacy JiT time). Use **`torch.float32`** (same as base iMF variants): + +```python +model_dir = Path("./iMF-B-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.float32, +).to("cuda") + +image = pipe( + class_labels="golden retriever", + num_inference_steps=1, + guidance_scale=8.0, + guidance_interval_start=0.4, + guidance_interval_end=0.65, + generator=torch.Generator(device="cuda").manual_seed(42), +).images[0] +``` + +## Inference (`iMF-L-SIM`) + +Same loading pattern as iMF-B-SIM (production `IMFPipeline` from `iMF-diffusers/iMF-L-2`): + +```python +model_dir = Path("./iMF-L-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.float32, +).to("cuda") + +image = pipe( + class_labels="golden retriever", + num_inference_steps=1, + guidance_scale=8.0, + guidance_interval_start=0.4, + guidance_interval_end=0.65, + generator=torch.Generator(device="cuda").manual_seed(42), +).images[0] +``` + +Regenerate from checkpoint: + +```bash +python _convert_imf_l_fd_sim.py +``` + +## Inference (`iMF-XL-SIM`) + +Same loading pattern as iMF-B/L-SIM (production `IMFPipeline` from `iMF-diffusers/iMF-XL-2`): + +```python +model_dir = Path("./iMF-XL-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.float32, +).to("cuda") + +image = pipe( + class_labels="golden retriever", + num_inference_steps=1, + guidance_scale=8.0, + guidance_interval_start=0.4, + guidance_interval_end=0.65, + generator=torch.Generator(device="cuda").manual_seed(42), +).images[0] +``` + + +## Inference (`pMF-B-16-SIM`) + +Uses production `PMFPipeline` from `pMF-diffusers/pMF-B-16` (native pMF time convention). Use **`torch.bfloat16`** on CUDA: + +```python +model_dir = Path("./pMF-B-16-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.bfloat16, +).to("cuda") + +image = pipe( + class_labels="golden retriever", + num_inference_steps=1, + guidance_scale=7.5, + guidance_interval_min=0.1, + guidance_interval_max=0.8, + noise_scale=1.0, + generator=torch.Generator(device="cuda").manual_seed(42), +).images[0] +``` + +## Inference (`pMF-B-32-SIM`) + +Same loading pattern as pMF-B-16-SIM (production `PMFPipeline` from `pMF-diffusers/pMF-B-32`): + +```python +model_dir = Path("./pMF-B-32-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.bfloat16, +).to("cuda") + +image = pipe( + class_labels="golden retriever", + num_inference_steps=1, + guidance_scale=6.5, + guidance_interval_min=0.1, + guidance_interval_max=0.7, + noise_scale=2.0, + generator=torch.Generator(device="cuda").manual_seed(42), +).images[0] +``` + diff --git a/iMF-B-SIM/README.md b/iMF-B-SIM/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a32770fbf128b9f2f394b83e2229b9cf51ddc7f0 --- /dev/null +++ b/iMF-B-SIM/README.md @@ -0,0 +1,40 @@ +--- +license: mit +library_name: diffusers +pipeline_tag: text-to-image +tags: +- diffusers +- imf +- image-generation +- class-conditional +--- + +# iMF-B-SIM + +Self-contained Diffusers variant for **iMF-B/2 (FD-SIM post-trained)** (Improved Mean Flows). + +## Load + +```python +from pathlib import Path +from diffusers import DiffusionPipeline +import torch + +model_dir = Path("iMF-B-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + local_files_only=True, + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.float32, +) +pipe.to("cuda") + +image = pipe( + class_labels=207, + num_inference_steps=1, + guidance_scale=8.0, + guidance_interval_start=0.4, + guidance_interval_end=0.65, +).images[0] +``` diff --git a/iMF-B-SIM/model_index.json b/iMF-B-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..80209bf0b9dd1de11a6a86528d2ebb23e3fe20c1 --- /dev/null +++ b/iMF-B-SIM/model_index.json @@ -0,0 +1,1021 @@ +{ + "_class_name": [ + "pipeline", + "IMFPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_imf", + "IMFScheduler" + ], + "transformer": [ + "transformer_imf", + "IMFTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + }, + "vae": [ + "diffusers", + "AutoencoderKL" + ] +} diff --git a/iMF-B-SIM/pipeline.py b/iMF-B-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..f137dbd473c0740293beff2648a0f78bf823ecce --- /dev/null +++ b/iMF-B-SIM/pipeline.py @@ -0,0 +1,342 @@ +"""Hub custom pipeline: IMFPipeline. +Load with native Hugging Face diffusers and trust_remote_code=True. +""" + +from __future__ import annotations + +import importlib.util +import inspect + +import json +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union, Any + +import torch +from diffusers.image_processor import VaeImageProcessor +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +# imeanflow / FD-Loss sd-vae latent statistics (stabilityai/sd-vae-ft-mse) +LATENT_CHANNEL_MEAN = (0.86488, -0.27787343, 0.21616915, 0.3738409) +LATENT_CHANNEL_STD = (4.85503674, 5.31922414, 3.93725398, 3.9870003) + + +def _load_bundled_vae(transformer) -> Optional[Any]: + transformer_path = getattr(transformer.config, "_name_or_path", None) + if not transformer_path: + return None + vae_dir = Path(transformer_path).resolve().parent / "vae" + if not vae_dir.is_dir() or not (vae_dir / "config.json").is_file(): + return None + from diffusers import AutoencoderKL + + vae_dtype = getattr(transformer, "dtype", torch.float32) + return AutoencoderKL.from_pretrained(str(vae_dir), torch_dtype=vae_dtype) + + +class IMFPipeline(DiffusionPipeline): + + @staticmethod + def prepare_extra_step_kwargs( + scheduler, + generator=None, + eta: float | None = None, + ): + kwargs = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _prepare_generator( + generator: Optional[Union[torch.Generator, List[torch.Generator]]], + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(generator, list): + for gen in generator: + if gen is not None: + gen.manual_seed(int(gen.initial_seed())) + return generator + generator.manual_seed(int(generator.initial_seed())) + return generator + + @staticmethod + def _coerce_scheduler(scheduler, transformer) -> Any: + if scheduler is not None and not isinstance(scheduler, (list, tuple)): + return scheduler + variant_path = getattr(transformer.config, "_name_or_path", None) + if variant_path: + scheduler_dir = Path(variant_path).resolve().parent / "scheduler" + module_path = scheduler_dir / "scheduling_imf.py" + config_path = scheduler_dir / "scheduler_config.json" + if module_path.is_file() and config_path.is_file(): + spec = importlib.util.spec_from_file_location("scheduling_imf", module_path) + if spec is not None and spec.loader is not None: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.IMFScheduler.from_pretrained(str(scheduler_dir)) + raise ValueError( + "IMFPipeline could not load IMFScheduler. Ensure the variant includes scheduler/scheduling_imf.py." + ) + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + model_cpu_offload_seq = "transformer->vae" + + def __init__( + self, + transformer, + scheduler, + vae=None, + id2label: Optional[Dict[Union[int, str], str]] = None, + ): + super().__init__() + scheduler = self._coerce_scheduler(scheduler, transformer) + if scheduler is None: + raise ValueError("IMFPipeline requires a scheduler loaded from the checkpoint.") + if isinstance(vae, (list, tuple)): + vae = None + if vae is None: + vae = _load_bundled_vae(transformer) + if vae is not None: + vae = vae.to(device=transformer.device, dtype=getattr(transformer, "dtype", torch.float32)) + self.register_modules(transformer=transformer, scheduler=scheduler, vae=vae) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + self.latent_channel_mean = torch.tensor(LATENT_CHANNEL_MEAN, dtype=torch.float32).view(1, 4, 1, 1) + self.latent_channel_std = torch.tensor(LATENT_CHANNEL_STD, dtype=torch.float32).view(1, 4, 1, 1) + vae_scale_factor = 8 + if vae is not None: + vae_scale_factor = 2 ** (len(vae.config.block_out_channels) - 1) + self.vae_scale_factor = vae_scale_factor + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + loaded = self._read_id2label_from_model_index(getattr(self.config, "_name_or_path", None)) + if loaded: + self._id2label = loaded + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _read_id2label_from_model_index(variant_path: Optional[str]) -> Dict[int, str]: + if not variant_path: + return {} + model_index_path = Path(variant_path).resolve() / "model_index.json" + if not model_index_path.exists(): + return {} + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if not isinstance(id2label, dict): + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + self._ensure_labels_loaded() + if not self.labels: + raise ValueError("No labels loaded. Ensure `id2label` exists in model_index.json.") + if isinstance(label, str): + label = [label] + missing = [item for item in label if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Example valid labels: {preview}, ...") + return [self.labels[item] for item in label] + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) + return list(class_labels) + + def _denormalize_latents(self, latents: torch.Tensor) -> torch.Tensor: + mean = self.latent_channel_mean.to(device=latents.device, dtype=latents.dtype) + std = self.latent_channel_std.to(device=latents.device, dtype=latents.dtype) + return latents * std + mean + + def decode_latents(self, latents: torch.Tensor, output_type: str = "pil"): + if output_type == "latent": + return latents + if self.vae is None: + raise ValueError( + "Cannot decode latents without a VAE. Ensure model_index.json lists vae and the variant includes vae/." + ) + vae_dtype = next(self.vae.parameters()).dtype + if next(self.vae.parameters()).device != latents.device: + self.vae.to(device=latents.device, dtype=vae_dtype) + latents = self._denormalize_latents(latents).to(dtype=vae_dtype) + image = self.vae.decode(latents).sample + return self.image_processor.postprocess(image, output_type=output_type) + + def _predict_velocity_u( + self, + latents: torch.Tensor, + timestep: torch.Tensor, + time_gap: torch.Tensor, + class_labels: torch.Tensor, + class_null: torch.Tensor, + guidance_scale: float, + guidance_interval_start: float, + guidance_interval_end: float, + do_classifier_free_guidance: bool, + ) -> torch.Tensor: + if do_classifier_free_guidance: + latents_in = torch.cat([latents, latents], dim=0) + labels = torch.cat([class_labels, class_null], dim=0) + omega = torch.tensor([guidance_scale, 1.0], device=latents.device, dtype=latents.dtype) + t_min = torch.tensor([guidance_interval_start, 0.0], device=latents.device, dtype=latents.dtype) + t_max = torch.tensor([guidance_interval_end, 1.0], device=latents.device, dtype=latents.dtype) + batch = latents.shape[0] + timestep_in = timestep.reshape(1).repeat(2 * batch).to(dtype=latents.dtype) + time_gap_in = time_gap.reshape(1).repeat(2 * batch).to(dtype=latents.dtype) + omega = omega.repeat(batch) + t_min = t_min.repeat(batch) + t_max = t_max.repeat(batch) + else: + latents_in = latents + labels = class_labels + batch = latents.shape[0] + timestep_in = timestep.reshape(1).repeat(batch).to(dtype=latents.dtype) + time_gap_in = time_gap.reshape(1).repeat(batch).to(dtype=latents.dtype) + omega = torch.full((batch,), guidance_scale, device=latents.device, dtype=latents.dtype) + t_min = torch.full((batch,), guidance_interval_start, device=latents.device, dtype=latents.dtype) + t_max = torch.full((batch,), guidance_interval_end, device=latents.device, dtype=latents.dtype) + + outputs = self.transformer( + sample=latents_in, + timestep=timestep_in, + class_labels=labels, + time_gap=time_gap_in, + guidance_scale=omega, + guidance_interval_start=t_min, + guidance_interval_end=t_max, + return_dict=True, + ) + velocity_u = outputs.velocity_u + + if not do_classifier_free_guidance: + return velocity_u + + u_cond, u_uncond = velocity_u.chunk(2, dim=0) + return u_uncond + guidance_scale * (u_cond - u_uncond) + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + num_inference_steps: int = 1, + guidance_scale: float = 2.7, + guidance_interval_start: float = 0.1, + guidance_interval_end: float = 0.9, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + if output_type not in {"pil", "np", "pt", "latent"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt', 'latent'.") + + class_label_ids = self._normalize_class_labels(class_labels) + do_classifier_free_guidance = guidance_scale > 1.0 + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + + image_size = int(self.transformer.config.sample_size) + channels = int(self.transformer.config.in_channels) + null_class_val = int(getattr(self.transformer.config, "num_classes", 1000)) + + generator = self._prepare_generator(generator) + + if latents is None: + latents = randn_tensor( + shape=(batch_size, channels, image_size, image_size), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) + + class_labels_t = torch.tensor(class_label_ids, device=latents.device, dtype=torch.long).reshape(-1) + class_labels_t = class_labels_t.clamp(0, null_class_val - 1) + class_null = torch.full_like(class_labels_t, null_class_val) + + self.scheduler.set_timesteps(num_inference_steps, device=latents.device) + timesteps = self.scheduler.timesteps + + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + + for i in self.progress_bar(range(num_inference_steps)): + t = timesteps[i] + t_next = timesteps[i + 1] + time_gap = t - t_next + velocity_u = self._predict_velocity_u( + latents, + t, + time_gap, + class_labels_t, + class_null, + guidance_scale, + guidance_interval_start, + guidance_interval_end, + do_classifier_free_guidance, + ) + latents = self.scheduler.step(velocity_u, t, latents, **extra_step_kwargs).prev_sample + + images = self.decode_latents(latents, output_type=output_type) + + self.maybe_free_model_hooks() + + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) + + +IMFPipelineOutput = ImagePipelineOutput diff --git a/iMF-B-SIM/scheduler/scheduler_config.json b/iMF-B-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c643eeffcc65000209b57865d0f68d2870d004a8 --- /dev/null +++ b/iMF-B-SIM/scheduler/scheduler_config.json @@ -0,0 +1,5 @@ +{ + "_class_name": "IMFScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000 +} diff --git a/iMF-B-SIM/scheduler/scheduling_imf.py b/iMF-B-SIM/scheduler/scheduling_imf.py new file mode 100644 index 0000000000000000000000000000000000000000..7dfaeeb301cf121a4ec6a24e4cf6ab5fe33fae6e --- /dev/null +++ b/iMF-B-SIM/scheduler/scheduling_imf.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class IMFSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class IMFScheduler(SchedulerMixin, ConfigMixin): + """Mean-flow scheduler with timesteps from 1.0 to 0.0.""" + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + shift: float = 1.0, + stochastic_sampling: bool = False, + ): + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> None: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.flatten()[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[IMFSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = (t - t_next).to(dtype=sample.dtype, device=sample.device) + while dt.ndim < sample.ndim: + dt = dt.unsqueeze(-1) + + if getattr(self.config, "stochastic_sampling", False): + t_bc = t.to(dtype=sample.dtype, device=sample.device) + while t_bc.ndim < sample.ndim: + t_bc = t_bc.unsqueeze(-1) + t_next_bc = t_next.to(dtype=sample.dtype, device=sample.device) + while t_next_bc.ndim < sample.ndim: + t_next_bc = t_next_bc.unsqueeze(-1) + x0 = sample - t_bc * model_output + noise = randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + prev_sample = (1.0 - t_next_bc) * x0 + t_next_bc * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return IMFSchedulerOutput(prev_sample=prev_sample) diff --git a/iMF-B-SIM/transformer/config.json b/iMF-B-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e88f49b3e040d10be722f55b25d8a5dce4ebfe26 --- /dev/null +++ b/iMF-B-SIM/transformer/config.json @@ -0,0 +1,23 @@ +{ + "_class_name": "IMFTransformer2DModel", + "_diffusers_version": "0.38.0", + "aux_head_depth": 8, + "depth": 12, + "embedding_init_constant": 1.0, + "eval_mode": true, + "hidden_size": 768, + "in_channels": 4, + "mlp_ratio": 2.6666666666666665, + "model_type": "iMF-B/2", + "num_attention_heads": 12, + "num_cfg_tokens": 4, + "num_class_embeds": null, + "num_class_tokens": 8, + "num_classes": 1000, + "num_interval_tokens": 2, + "num_time_tokens": 4, + "patch_size": 2, + "sample_size": 32, + "token_init_constant": 1.0, + "weight_init_constant": 0.32 +} diff --git a/iMF-B-SIM/transformer/diffusion_pytorch_model.safetensors b/iMF-B-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..84af6732e445e1db9d95bdc12bdda27b0c74e73c --- /dev/null +++ b/iMF-B-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:583dca284df978363872471904c2b86bc517b8d821b4513f0c304130044ecc6a +size 355811968 diff --git a/iMF-B-SIM/transformer/transformer_imf.py b/iMF-B-SIM/transformer/transformer_imf.py new file mode 100644 index 0000000000000000000000000000000000000000..78243475f280c1487713c67d2340cf22ac6249c0 --- /dev/null +++ b/iMF-B-SIM/transformer/transformer_imf.py @@ -0,0 +1,586 @@ +from __future__ import annotations + +import argparse +import math +from collections.abc import Mapping +from dataclasses import dataclass +from functools import partial +from math import sqrt +from typing import Dict, Literal, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils import BaseOutput + + +IMF_PRESET_CONFIGS: Dict[str, Dict[str, object]] = { + "iMF-B/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 768, + "depth": 12, + "num_attention_heads": 12, + "aux_head_depth": 8, + }, + "iMF-M/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 768, + "depth": 24, + "num_attention_heads": 12, + "aux_head_depth": 8, + }, + "iMF-L/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "aux_head_depth": 8, + }, + "iMF-XL/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 1024, + "depth": 48, + "num_attention_heads": 16, + "aux_head_depth": 8, + }, +} + + +@dataclass +class IMFTransformer2DModelOutput(BaseOutput): + velocity_u: torch.Tensor + velocity_v: Optional[torch.Tensor] = None + + +def remap_legacy_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Map legacy iMF / Flax-style keys to native IMFTransformer2DModel keys.""" + remapped: Dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + for prefix in ("net.", "transformer."): + if new_key.startswith(prefix): + new_key = new_key[len(prefix) :] + break + new_key = new_key.replace(".kernel", ".weight") + new_key = new_key.replace("._flax_linear.", ".linear.") + new_key = new_key.replace("._flax_embedding.", ".embedding.") + remapped[new_key] = value + return remapped + + +def config_from_legacy(config: Dict[str, object]) -> Dict[str, object]: + model_type = config.get("model_type") or config.get("model_str") or config.get("model_name") + if model_type not in IMF_PRESET_CONFIGS: + raise ValueError(f"Unknown iMF preset '{model_type}'. Known: {list(IMF_PRESET_CONFIGS)}") + + preset = dict(IMF_PRESET_CONFIGS[model_type]) + preset["num_classes"] = int(config.get("num_class_embeds") or config.get("num_classes") or 1000) + preset["model_type"] = model_type + if config.get("sample_size") is not None: + preset["sample_size"] = int(config["sample_size"]) + if config.get("in_channels") is not None: + preset["in_channels"] = int(config["in_channels"]) + return preset + + +def _unsqueeze(t: torch.Tensor, dim: int) -> torch.Tensor: + return t.unsqueeze(dim) + + +class _ScaledLinear(nn.Module): + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + bias_init: str = "zeros", + ): + super().__init__() + self.linear = nn.Linear(in_features, out_features, bias=bias) + if weight_init == "scaled_variance": + std = init_constant / sqrt(in_features) + nn.init.normal_(self.linear.weight, std=std) + elif weight_init == "zeros": + nn.init.zeros_(self.linear.weight) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + if bias: + if bias_init == "zeros": + nn.init.zeros_(self.linear.bias) + else: + raise ValueError(f"Invalid bias_init: {bias_init}") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(x) + + +class _ScaledEmbedding(nn.Module): + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + ): + super().__init__() + self.embedding = nn.Embedding(num_embeddings, embedding_dim) + if weight_init == "scaled_variance": + std = init_constant / sqrt(embedding_dim) + nn.init.normal_(self.embedding.weight, std=std) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.embedding(x) + + +class _RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.eps = eps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + mean_square = torch.mean(torch.square(x), dim=-1, keepdim=True) + output = x * torch.rsqrt(mean_square + self.eps) + return output.to(x.dtype) * self.weight + + +class _SwiGLUMlp(nn.Module): + def __init__( + self, + in_features: int, + hidden_features: int, + weight_init: str = "scaled_variance", + weight_init_constant: float = 1.0, + ): + super().__init__() + init_kwargs = dict(bias=False, weight_init=weight_init, init_constant=weight_init_constant) + self.w1 = _ScaledLinear(in_features, hidden_features, **init_kwargs) + self.w3 = _ScaledLinear(in_features, hidden_features, **init_kwargs) + self.w2 = _ScaledLinear(hidden_features, in_features, **init_kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + + +class IMFTimestepEmbedder(nn.Module): + def __init__( + self, + hidden_size: int, + frequency_embedding_size: int = 256, + init_constant: float = 1.0, + ): + super().__init__() + self.mlp = nn.Sequential( + _ScaledLinear(frequency_embedding_size, hidden_size, bias=True, init_constant=init_constant), + nn.SiLU(), + _ScaledLinear(hidden_size, hidden_size, bias=True, init_constant=init_constant), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) * torch.arange(start=0, end=half, dtype=t.dtype, device=t.device) / half + ) + args = t[:, None] * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + return self.mlp(self.timestep_embedding(t, self.frequency_embedding_size)) + + +class IMFLabelEmbedder(nn.Module): + def __init__(self, num_classes: int, hidden_size: int, init_constant: float = 1.0): + super().__init__() + self.embedding_table = _ScaledEmbedding(num_classes + 1, hidden_size, init_constant=init_constant) + + def forward(self, labels: torch.Tensor) -> torch.Tensor: + return self.embedding_table(labels) + + +class IMFPatchEmbedder(nn.Module): + def __init__(self, input_size: int, patch_size: int, in_channels: int, hidden_size: int, bias: bool = True): + super().__init__() + self.patch_size = (patch_size, patch_size) + self.num_patches = (input_size // patch_size) ** 2 + self.proj = nn.Conv2d( + in_channels, + hidden_size, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + ) + kh = kw = patch_size + fan_in = kh * kw * in_channels + fan_out = hidden_size + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj.weight, -limit, limit) + if bias: + nn.init.zeros_(self.proj.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj(x) + return x.flatten(2).transpose(1, 2) + + +def precompute_rope_freqs(dim: int, seq_len: int, theta: float = 10000.0, device: torch.device | None = None): + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim)) + positions = torch.arange(seq_len, dtype=torch.float32, device=device) + freqs_cis = torch.outer(positions, freqs) + return torch.complex(torch.cos(freqs_cis), torch.sin(freqs_cis)) + + +def apply_rotary_pos_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + x_dtype = x.dtype + x_complex = torch.view_as_complex(x.to(torch.float32).reshape(*x.shape[:-1], -1, 2)) + freqs_cis = freqs_cis.unsqueeze(0).unsqueeze(2).to(x.device) + x_rotated = x_complex * freqs_cis + x_out = torch.view_as_real(x_rotated).flatten(-2) + return x_out.to(dtype=x_dtype) + + +class IMFRoPEAttention(nn.Module): + def __init__(self, hidden_size: int, num_heads: int, weight_init_constant: float = 0.32): + super().__init__() + self.hidden_size = hidden_size + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + init_kwargs = dict(bias=False, init_constant=weight_init_constant) + self.q_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.k_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.v_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.out_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.q_norm = _RMSNorm(self.head_dim) + self.k_norm = _RMSNorm(self.head_dim) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + batch, seq_len, _ = x.shape + q = self.q_proj(x).reshape(batch, seq_len, self.num_heads, self.head_dim) + k = self.k_proj(x).reshape(batch, seq_len, self.num_heads, self.head_dim) + v = self.v_proj(x).reshape(batch, seq_len, self.num_heads, self.head_dim) + q = apply_rotary_pos_emb(self.q_norm(q), rope_freqs) + k = apply_rotary_pos_emb(self.k_norm(k), rope_freqs) + query = q / math.sqrt(self.head_dim) + attn_weights = torch.einsum("bqhd,bkhd->bhqk", query, k) + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32) + attn = torch.einsum("bhqk,bkhd->bqhd", attn_weights.to(dtype=v.dtype), v) + return self.out_proj(attn.reshape(batch, seq_len, self.hidden_size)) + + +class IMFTransformerBlock(nn.Module): + def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float, weight_init_constant: float): + super().__init__() + self.norm1 = _RMSNorm(hidden_size) + self.attn = IMFRoPEAttention(hidden_size, num_heads, weight_init_constant=weight_init_constant) + self.norm2 = _RMSNorm(hidden_size) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + self.mlp = _SwiGLUMlp(hidden_size, mlp_hidden_dim, weight_init_constant=weight_init_constant) + self.attn_scale = nn.Parameter(torch.zeros(hidden_size)) + self.mlp_scale = nn.Parameter(torch.zeros(hidden_size)) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), rope_freqs) * self.attn_scale + x = x + self.mlp(self.norm2(x)) * self.mlp_scale + return x + + +class IMFFinalLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int): + super().__init__() + self.norm = _RMSNorm(hidden_size) + self.linear = _ScaledLinear( + hidden_size, + patch_size * patch_size * out_channels, + bias=True, + weight_init="zeros", + init_constant=1.0, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(self.norm(x)) + + +class IMFTransformer2DModel(ModelMixin, ConfigMixin): + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: int = 32, + patch_size: int = 2, + in_channels: int = 4, + hidden_size: int = 768, + depth: int = 12, + num_attention_heads: int = 12, + mlp_ratio: float = 8 / 3, + num_classes: int = 1000, + aux_head_depth: int = 8, + num_class_tokens: int = 8, + num_time_tokens: int = 4, + num_cfg_tokens: int = 4, + num_interval_tokens: int = 2, + token_init_constant: float = 1.0, + embedding_init_constant: float = 1.0, + weight_init_constant: float = 0.32, + eval_mode: bool = True, + model_type: str | None = None, + num_class_embeds: int | None = None, + ): + super().__init__() + if num_class_embeds is not None: + num_classes = int(num_class_embeds) + if model_type in IMF_PRESET_CONFIGS: + preset = IMF_PRESET_CONFIGS[model_type] + sample_size = int(preset["sample_size"]) + patch_size = int(preset["patch_size"]) + hidden_size = int(preset["hidden_size"]) + depth = int(preset["depth"]) + num_attention_heads = int(preset["num_attention_heads"]) + aux_head_depth = int(preset["aux_head_depth"]) + + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.depth = depth + self.num_attention_heads = num_attention_heads + self.num_classes = num_classes + self.aux_head_depth = aux_head_depth + self.num_class_tokens = num_class_tokens + self.num_time_tokens = num_time_tokens + self.num_cfg_tokens = num_cfg_tokens + self.num_interval_tokens = num_interval_tokens + self.eval_mode = eval_mode + self.gradient_checkpointing = False + + self.x_embedder = IMFPatchEmbedder(sample_size, patch_size, in_channels, hidden_size, bias=True) + embed_kwargs = dict(hidden_size=hidden_size, init_constant=embedding_init_constant) + self.h_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.omega_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_start_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_end_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.y_embedder = IMFLabelEmbedder(num_classes, hidden_size, init_constant=embedding_init_constant) + + token_std = token_init_constant / math.sqrt(hidden_size) + self.time_tokens = nn.Parameter(torch.randn(num_time_tokens, hidden_size) * token_std) + self.class_tokens = nn.Parameter(torch.randn(num_class_tokens, hidden_size) * token_std) + self.omega_tokens = nn.Parameter(torch.randn(num_cfg_tokens, hidden_size) * token_std) + self.t_min_tokens = nn.Parameter(torch.randn(num_interval_tokens, hidden_size) * token_std) + self.t_max_tokens = nn.Parameter(torch.randn(num_interval_tokens, hidden_size) * token_std) + + total_tokens = ( + self.x_embedder.num_patches + + num_class_tokens + + num_cfg_tokens + + 2 * num_interval_tokens + + num_time_tokens + ) + self.prefix_tokens = num_class_tokens + num_cfg_tokens + 2 * num_interval_tokens + num_time_tokens + self.head_dim = hidden_size // num_attention_heads + self.register_buffer( + "rope_freqs", + precompute_rope_freqs(self.head_dim, total_tokens), + persistent=False, + ) + + shared_depth = depth - aux_head_depth + block_kwargs = dict( + hidden_size=hidden_size, + num_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + weight_init_constant=weight_init_constant, + ) + self.shared_blocks = nn.ModuleList([IMFTransformerBlock(**block_kwargs) for _ in range(shared_depth)]) + self.u_heads = nn.ModuleList([IMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth)]) + self.v_heads = nn.ModuleList( + [IMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth if not eval_mode else 0)] + ) + self.u_final_layer = IMFFinalLayer(hidden_size, patch_size, in_channels) + self.v_final_layer = IMFFinalLayer(hidden_size, patch_size, in_channels) + + def unpatchify(self, x: torch.Tensor) -> torch.Tensor: + c = self.out_channels + p = self.patch_size + h = w = int(x.shape[1] ** 0.5) + x = x.reshape(x.shape[0], h, w, p, p, c) + x = torch.einsum("nhwpqc->nchpwq", x) + return x.reshape(x.shape[0], c, h * p, w * p) + + def _build_sequence( + self, + sample: torch.Tensor, + time_gap: torch.Tensor, + guidance_scale: torch.Tensor, + guidance_interval_start: torch.Tensor, + guidance_interval_end: torch.Tensor, + class_labels: torch.Tensor, + ) -> torch.Tensor: + x_embed = self.x_embedder(sample) + h_embed = self.h_embedder(time_gap) + omega_embed = self.omega_embedder(1 - 1 / guidance_scale) + t_min_embed = self.cfg_t_start_embedder(guidance_interval_start) + t_max_embed = self.cfg_t_end_embedder(guidance_interval_end) + y_embed = self.y_embedder(class_labels) + + time_tokens = self.time_tokens + _unsqueeze(h_embed, 1) + omega_tokens = self.omega_tokens + _unsqueeze(omega_embed, 1) + t_min_tokens = self.t_min_tokens + _unsqueeze(t_min_embed, 1) + t_max_tokens = self.t_max_tokens + _unsqueeze(t_max_embed, 1) + class_tokens = self.class_tokens + _unsqueeze(y_embed, 1) + + return torch.cat( + [class_tokens, omega_tokens, t_min_tokens, t_max_tokens, time_tokens, x_embed], + dim=1, + ) + + def forward( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + time_gap: Optional[torch.Tensor] = None, + guidance_scale: Optional[torch.Tensor] = None, + guidance_interval_start: Optional[torch.Tensor] = None, + guidance_interval_end: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[IMFTransformer2DModelOutput, Tuple[torch.Tensor, ...]]: + batch_size = sample.shape[0] + timestep = self._expand_batch(timestep, batch_size, sample.device, sample.dtype) + if time_gap is None: + time_gap = timestep + else: + time_gap = self._expand_batch(time_gap, batch_size, sample.device, sample.dtype) + if guidance_scale is None: + guidance_scale = torch.ones(batch_size, device=sample.device, dtype=sample.dtype) + else: + guidance_scale = self._expand_batch(guidance_scale, batch_size, sample.device, sample.dtype) + if guidance_interval_start is None: + guidance_interval_start = torch.zeros(batch_size, device=sample.device, dtype=sample.dtype) + else: + guidance_interval_start = self._expand_batch( + guidance_interval_start, batch_size, sample.device, sample.dtype + ) + if guidance_interval_end is None: + guidance_interval_end = torch.ones(batch_size, device=sample.device, dtype=sample.dtype) + else: + guidance_interval_end = self._expand_batch( + guidance_interval_end, batch_size, sample.device, sample.dtype + ) + + class_labels = class_labels.reshape(-1).long() + seq = self._build_sequence( + sample, + time_gap, + guidance_scale, + guidance_interval_start, + guidance_interval_end, + class_labels, + ) + + for block in self.shared_blocks: + if self.training and self.gradient_checkpointing: + seq = torch.utils.checkpoint.checkpoint(block, seq, self.rope_freqs, use_reentrant=False) + else: + seq = block(seq, self.rope_freqs) + + u_seq = v_seq = seq + for block in self.u_heads: + u_seq = block(u_seq, self.rope_freqs) + for block in self.v_heads: + v_seq = block(v_seq, self.rope_freqs) + + u_tokens = u_seq[:, self.prefix_tokens :] + velocity_u = self.unpatchify(self.u_final_layer(u_tokens)) + + velocity_v = None + if len(self.v_heads) > 0: + v_tokens = v_seq[:, self.prefix_tokens :] + velocity_v = self.unpatchify(self.v_final_layer(v_tokens)) + + if not return_dict: + if velocity_v is None: + return (velocity_u,) + return (velocity_u, velocity_v) + return IMFTransformer2DModelOutput(velocity_u=velocity_u, velocity_v=velocity_v) + + @staticmethod + def _expand_batch( + value: torch.Tensor, + batch_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + value = torch.as_tensor(value, device=device, dtype=dtype) + if value.ndim == 0: + value = value.reshape(1).repeat(batch_size) + else: + value = value.reshape(-1) + if value.shape[0] == 1 and batch_size > 1: + value = value.repeat(batch_size) + return value + + @classmethod + def from_imf_checkpoint( + cls, + checkpoint_path: str, + model_type: str | None = None, + map_location: str = "cpu", + strict: bool = True, + eval_mode: bool = True, + ) -> Tuple["IMFTransformer2DModel", Dict[str, object]]: + checkpoint = torch.load(checkpoint_path, map_location=map_location, weights_only=False) + if isinstance(checkpoint, Mapping) and "state_dict" in checkpoint: + state_dict = checkpoint["state_dict"] + elif isinstance(checkpoint, Mapping) and any(k.startswith("net.") for k in checkpoint): + state_dict = checkpoint + else: + state_dict = checkpoint + + inferred_type = model_type + if inferred_type is None and isinstance(checkpoint, Mapping): + args = checkpoint.get("args") + if isinstance(args, argparse.Namespace): + inferred_type = getattr(args, "model", None) or getattr(args, "model_str", None) + elif isinstance(args, Mapping): + inferred_type = args.get("model") or args.get("model_str") + + if inferred_type is None: + raise ValueError("model_type must be provided when loading a raw state dict checkpoint.") + + config = dict(IMF_PRESET_CONFIGS[inferred_type]) + config["model_type"] = inferred_type + config["eval_mode"] = eval_mode + model = cls(**config) + model.load_state_dict(remap_legacy_state_dict(state_dict), strict=strict) + metadata = {"checkpoint_path": checkpoint_path, "model_type": inferred_type} + return model, metadata + + def to_imf_checkpoint(self, prefix: str = "net.") -> Dict[str, torch.Tensor]: + state: Dict[str, torch.Tensor] = {} + for key, value in self.state_dict().items(): + if key == "rope_freqs": + continue + state[f"{prefix}{key}"] = value.detach().cpu() + return state + + @property + def net(self): + return self + + +IMFDiffusersModel = IMFTransformer2DModel diff --git a/iMF-B-SIM/vae/config.json b/iMF-B-SIM/vae/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ae2ab37de2acb8bc348d3de809ae8385324be962 --- /dev/null +++ b/iMF-B-SIM/vae/config.json @@ -0,0 +1,38 @@ +{ + "_class_name": "AutoencoderKL", + "_diffusers_version": "0.36.0", + "_name_or_path": "stabilityai/sd-vae-ft-mse", + "act_fn": "silu", + "block_out_channels": [ + 128, + 256, + 512, + 512 + ], + "down_block_types": [ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D" + ], + "force_upcast": true, + "in_channels": 3, + "latent_channels": 4, + "latents_mean": null, + "latents_std": null, + "layers_per_block": 2, + "mid_block_add_attention": true, + "norm_num_groups": 32, + "out_channels": 3, + "sample_size": 256, + "scaling_factor": 0.18215, + "shift_factor": null, + "up_block_types": [ + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D" + ], + "use_post_quant_conv": true, + "use_quant_conv": true +} diff --git a/iMF-B-SIM/vae/diffusion_pytorch_model.safetensors b/iMF-B-SIM/vae/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..90464d67ac7303d0ee4696334df13da130a948ea --- /dev/null +++ b/iMF-B-SIM/vae/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d993488569e928462932c8c38a0760b874d166399b14414135bd9c42df5815 +size 334643276 diff --git a/iMF-L-SIM/README.md b/iMF-L-SIM/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ff06f1d15d27bb1e1f16b45b7c57c6ce8099ca7c --- /dev/null +++ b/iMF-L-SIM/README.md @@ -0,0 +1,40 @@ +--- +license: mit +library_name: diffusers +pipeline_tag: text-to-image +tags: +- diffusers +- imf +- image-generation +- class-conditional +--- + +# iMF-L-SIM + +Self-contained Diffusers variant for **iMF-L/2 (FD-SIM post-trained)** (Improved Mean Flows). + +## Load + +```python +from pathlib import Path +from diffusers import DiffusionPipeline +import torch + +model_dir = Path("iMF-L-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + local_files_only=True, + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.float32, +) +pipe.to("cuda") + +image = pipe( + class_labels=207, + num_inference_steps=1, + guidance_scale=8.0, + guidance_interval_start=0.4, + guidance_interval_end=0.65, +).images[0] +``` diff --git a/iMF-L-SIM/model_index.json b/iMF-L-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..80209bf0b9dd1de11a6a86528d2ebb23e3fe20c1 --- /dev/null +++ b/iMF-L-SIM/model_index.json @@ -0,0 +1,1021 @@ +{ + "_class_name": [ + "pipeline", + "IMFPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_imf", + "IMFScheduler" + ], + "transformer": [ + "transformer_imf", + "IMFTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + }, + "vae": [ + "diffusers", + "AutoencoderKL" + ] +} diff --git a/iMF-L-SIM/pipeline.py b/iMF-L-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..f137dbd473c0740293beff2648a0f78bf823ecce --- /dev/null +++ b/iMF-L-SIM/pipeline.py @@ -0,0 +1,342 @@ +"""Hub custom pipeline: IMFPipeline. +Load with native Hugging Face diffusers and trust_remote_code=True. +""" + +from __future__ import annotations + +import importlib.util +import inspect + +import json +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union, Any + +import torch +from diffusers.image_processor import VaeImageProcessor +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +# imeanflow / FD-Loss sd-vae latent statistics (stabilityai/sd-vae-ft-mse) +LATENT_CHANNEL_MEAN = (0.86488, -0.27787343, 0.21616915, 0.3738409) +LATENT_CHANNEL_STD = (4.85503674, 5.31922414, 3.93725398, 3.9870003) + + +def _load_bundled_vae(transformer) -> Optional[Any]: + transformer_path = getattr(transformer.config, "_name_or_path", None) + if not transformer_path: + return None + vae_dir = Path(transformer_path).resolve().parent / "vae" + if not vae_dir.is_dir() or not (vae_dir / "config.json").is_file(): + return None + from diffusers import AutoencoderKL + + vae_dtype = getattr(transformer, "dtype", torch.float32) + return AutoencoderKL.from_pretrained(str(vae_dir), torch_dtype=vae_dtype) + + +class IMFPipeline(DiffusionPipeline): + + @staticmethod + def prepare_extra_step_kwargs( + scheduler, + generator=None, + eta: float | None = None, + ): + kwargs = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _prepare_generator( + generator: Optional[Union[torch.Generator, List[torch.Generator]]], + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(generator, list): + for gen in generator: + if gen is not None: + gen.manual_seed(int(gen.initial_seed())) + return generator + generator.manual_seed(int(generator.initial_seed())) + return generator + + @staticmethod + def _coerce_scheduler(scheduler, transformer) -> Any: + if scheduler is not None and not isinstance(scheduler, (list, tuple)): + return scheduler + variant_path = getattr(transformer.config, "_name_or_path", None) + if variant_path: + scheduler_dir = Path(variant_path).resolve().parent / "scheduler" + module_path = scheduler_dir / "scheduling_imf.py" + config_path = scheduler_dir / "scheduler_config.json" + if module_path.is_file() and config_path.is_file(): + spec = importlib.util.spec_from_file_location("scheduling_imf", module_path) + if spec is not None and spec.loader is not None: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.IMFScheduler.from_pretrained(str(scheduler_dir)) + raise ValueError( + "IMFPipeline could not load IMFScheduler. Ensure the variant includes scheduler/scheduling_imf.py." + ) + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + model_cpu_offload_seq = "transformer->vae" + + def __init__( + self, + transformer, + scheduler, + vae=None, + id2label: Optional[Dict[Union[int, str], str]] = None, + ): + super().__init__() + scheduler = self._coerce_scheduler(scheduler, transformer) + if scheduler is None: + raise ValueError("IMFPipeline requires a scheduler loaded from the checkpoint.") + if isinstance(vae, (list, tuple)): + vae = None + if vae is None: + vae = _load_bundled_vae(transformer) + if vae is not None: + vae = vae.to(device=transformer.device, dtype=getattr(transformer, "dtype", torch.float32)) + self.register_modules(transformer=transformer, scheduler=scheduler, vae=vae) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + self.latent_channel_mean = torch.tensor(LATENT_CHANNEL_MEAN, dtype=torch.float32).view(1, 4, 1, 1) + self.latent_channel_std = torch.tensor(LATENT_CHANNEL_STD, dtype=torch.float32).view(1, 4, 1, 1) + vae_scale_factor = 8 + if vae is not None: + vae_scale_factor = 2 ** (len(vae.config.block_out_channels) - 1) + self.vae_scale_factor = vae_scale_factor + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + loaded = self._read_id2label_from_model_index(getattr(self.config, "_name_or_path", None)) + if loaded: + self._id2label = loaded + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _read_id2label_from_model_index(variant_path: Optional[str]) -> Dict[int, str]: + if not variant_path: + return {} + model_index_path = Path(variant_path).resolve() / "model_index.json" + if not model_index_path.exists(): + return {} + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if not isinstance(id2label, dict): + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + self._ensure_labels_loaded() + if not self.labels: + raise ValueError("No labels loaded. Ensure `id2label` exists in model_index.json.") + if isinstance(label, str): + label = [label] + missing = [item for item in label if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Example valid labels: {preview}, ...") + return [self.labels[item] for item in label] + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) + return list(class_labels) + + def _denormalize_latents(self, latents: torch.Tensor) -> torch.Tensor: + mean = self.latent_channel_mean.to(device=latents.device, dtype=latents.dtype) + std = self.latent_channel_std.to(device=latents.device, dtype=latents.dtype) + return latents * std + mean + + def decode_latents(self, latents: torch.Tensor, output_type: str = "pil"): + if output_type == "latent": + return latents + if self.vae is None: + raise ValueError( + "Cannot decode latents without a VAE. Ensure model_index.json lists vae and the variant includes vae/." + ) + vae_dtype = next(self.vae.parameters()).dtype + if next(self.vae.parameters()).device != latents.device: + self.vae.to(device=latents.device, dtype=vae_dtype) + latents = self._denormalize_latents(latents).to(dtype=vae_dtype) + image = self.vae.decode(latents).sample + return self.image_processor.postprocess(image, output_type=output_type) + + def _predict_velocity_u( + self, + latents: torch.Tensor, + timestep: torch.Tensor, + time_gap: torch.Tensor, + class_labels: torch.Tensor, + class_null: torch.Tensor, + guidance_scale: float, + guidance_interval_start: float, + guidance_interval_end: float, + do_classifier_free_guidance: bool, + ) -> torch.Tensor: + if do_classifier_free_guidance: + latents_in = torch.cat([latents, latents], dim=0) + labels = torch.cat([class_labels, class_null], dim=0) + omega = torch.tensor([guidance_scale, 1.0], device=latents.device, dtype=latents.dtype) + t_min = torch.tensor([guidance_interval_start, 0.0], device=latents.device, dtype=latents.dtype) + t_max = torch.tensor([guidance_interval_end, 1.0], device=latents.device, dtype=latents.dtype) + batch = latents.shape[0] + timestep_in = timestep.reshape(1).repeat(2 * batch).to(dtype=latents.dtype) + time_gap_in = time_gap.reshape(1).repeat(2 * batch).to(dtype=latents.dtype) + omega = omega.repeat(batch) + t_min = t_min.repeat(batch) + t_max = t_max.repeat(batch) + else: + latents_in = latents + labels = class_labels + batch = latents.shape[0] + timestep_in = timestep.reshape(1).repeat(batch).to(dtype=latents.dtype) + time_gap_in = time_gap.reshape(1).repeat(batch).to(dtype=latents.dtype) + omega = torch.full((batch,), guidance_scale, device=latents.device, dtype=latents.dtype) + t_min = torch.full((batch,), guidance_interval_start, device=latents.device, dtype=latents.dtype) + t_max = torch.full((batch,), guidance_interval_end, device=latents.device, dtype=latents.dtype) + + outputs = self.transformer( + sample=latents_in, + timestep=timestep_in, + class_labels=labels, + time_gap=time_gap_in, + guidance_scale=omega, + guidance_interval_start=t_min, + guidance_interval_end=t_max, + return_dict=True, + ) + velocity_u = outputs.velocity_u + + if not do_classifier_free_guidance: + return velocity_u + + u_cond, u_uncond = velocity_u.chunk(2, dim=0) + return u_uncond + guidance_scale * (u_cond - u_uncond) + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + num_inference_steps: int = 1, + guidance_scale: float = 2.7, + guidance_interval_start: float = 0.1, + guidance_interval_end: float = 0.9, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + if output_type not in {"pil", "np", "pt", "latent"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt', 'latent'.") + + class_label_ids = self._normalize_class_labels(class_labels) + do_classifier_free_guidance = guidance_scale > 1.0 + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + + image_size = int(self.transformer.config.sample_size) + channels = int(self.transformer.config.in_channels) + null_class_val = int(getattr(self.transformer.config, "num_classes", 1000)) + + generator = self._prepare_generator(generator) + + if latents is None: + latents = randn_tensor( + shape=(batch_size, channels, image_size, image_size), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) + + class_labels_t = torch.tensor(class_label_ids, device=latents.device, dtype=torch.long).reshape(-1) + class_labels_t = class_labels_t.clamp(0, null_class_val - 1) + class_null = torch.full_like(class_labels_t, null_class_val) + + self.scheduler.set_timesteps(num_inference_steps, device=latents.device) + timesteps = self.scheduler.timesteps + + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + + for i in self.progress_bar(range(num_inference_steps)): + t = timesteps[i] + t_next = timesteps[i + 1] + time_gap = t - t_next + velocity_u = self._predict_velocity_u( + latents, + t, + time_gap, + class_labels_t, + class_null, + guidance_scale, + guidance_interval_start, + guidance_interval_end, + do_classifier_free_guidance, + ) + latents = self.scheduler.step(velocity_u, t, latents, **extra_step_kwargs).prev_sample + + images = self.decode_latents(latents, output_type=output_type) + + self.maybe_free_model_hooks() + + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) + + +IMFPipelineOutput = ImagePipelineOutput diff --git a/iMF-L-SIM/scheduler/scheduler_config.json b/iMF-L-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c643eeffcc65000209b57865d0f68d2870d004a8 --- /dev/null +++ b/iMF-L-SIM/scheduler/scheduler_config.json @@ -0,0 +1,5 @@ +{ + "_class_name": "IMFScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000 +} diff --git a/iMF-L-SIM/scheduler/scheduling_imf.py b/iMF-L-SIM/scheduler/scheduling_imf.py new file mode 100644 index 0000000000000000000000000000000000000000..7dfaeeb301cf121a4ec6a24e4cf6ab5fe33fae6e --- /dev/null +++ b/iMF-L-SIM/scheduler/scheduling_imf.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class IMFSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class IMFScheduler(SchedulerMixin, ConfigMixin): + """Mean-flow scheduler with timesteps from 1.0 to 0.0.""" + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + shift: float = 1.0, + stochastic_sampling: bool = False, + ): + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> None: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.flatten()[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[IMFSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = (t - t_next).to(dtype=sample.dtype, device=sample.device) + while dt.ndim < sample.ndim: + dt = dt.unsqueeze(-1) + + if getattr(self.config, "stochastic_sampling", False): + t_bc = t.to(dtype=sample.dtype, device=sample.device) + while t_bc.ndim < sample.ndim: + t_bc = t_bc.unsqueeze(-1) + t_next_bc = t_next.to(dtype=sample.dtype, device=sample.device) + while t_next_bc.ndim < sample.ndim: + t_next_bc = t_next_bc.unsqueeze(-1) + x0 = sample - t_bc * model_output + noise = randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + prev_sample = (1.0 - t_next_bc) * x0 + t_next_bc * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return IMFSchedulerOutput(prev_sample=prev_sample) diff --git a/iMF-L-SIM/transformer/config.json b/iMF-L-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5831f236c7f070efe272256349e3cd6c26ebf314 --- /dev/null +++ b/iMF-L-SIM/transformer/config.json @@ -0,0 +1,23 @@ +{ + "_class_name": "IMFTransformer2DModel", + "_diffusers_version": "0.38.0", + "aux_head_depth": 8, + "depth": 32, + "embedding_init_constant": 1.0, + "eval_mode": true, + "hidden_size": 1024, + "in_channels": 4, + "mlp_ratio": 2.6666666666666665, + "model_type": "iMF-L/2", + "num_attention_heads": 16, + "num_cfg_tokens": 4, + "num_class_embeds": null, + "num_class_tokens": 8, + "num_classes": 1000, + "num_interval_tokens": 2, + "num_time_tokens": 4, + "patch_size": 2, + "sample_size": 32, + "token_init_constant": 1.0, + "weight_init_constant": 0.32 +} diff --git a/iMF-L-SIM/transformer/diffusion_pytorch_model.safetensors b/iMF-L-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..52d8bfbf65d5a66df98754a57aa2d4fee95178d2 --- /dev/null +++ b/iMF-L-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:908b20e09e463897734a3735b98dce730f6b050c9612314972f0a0a04ed66acc +size 1636334400 diff --git a/iMF-L-SIM/transformer/transformer_imf.py b/iMF-L-SIM/transformer/transformer_imf.py new file mode 100644 index 0000000000000000000000000000000000000000..be1783e9bd8ae5b81a82c8d01640e0d214004c26 --- /dev/null +++ b/iMF-L-SIM/transformer/transformer_imf.py @@ -0,0 +1,586 @@ +from __future__ import annotations + +import argparse +import math +from collections.abc import Mapping +from dataclasses import dataclass +from functools import partial +from math import sqrt +from typing import Dict, Literal, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils import BaseOutput + + +IMF_PRESET_CONFIGS: Dict[str, Dict[str, object]] = { + "iMF-B/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 768, + "depth": 12, + "num_attention_heads": 12, + "aux_head_depth": 8, + }, + "iMF-M/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 768, + "depth": 24, + "num_attention_heads": 12, + "aux_head_depth": 8, + }, + "iMF-L/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "aux_head_depth": 8, + }, + "iMF-XL/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 1024, + "depth": 48, + "num_attention_heads": 16, + "aux_head_depth": 8, + }, +} + + +@dataclass +class IMFTransformer2DModelOutput(BaseOutput): + velocity_u: torch.Tensor + velocity_v: Optional[torch.Tensor] = None + + +def remap_legacy_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Map legacy iMF / Flax-style keys to native IMFTransformer2DModel keys.""" + remapped: Dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + for prefix in ("net.", "transformer."): + if new_key.startswith(prefix): + new_key = new_key[len(prefix) :] + break + new_key = new_key.replace(".kernel", ".weight") + new_key = new_key.replace("._flax_linear.", ".linear.") + new_key = new_key.replace("._flax_embedding.", ".embedding.") + remapped[new_key] = value + return remapped + + +def config_from_legacy(config: Dict[str, object]) -> Dict[str, object]: + model_type = config.get("model_type") or config.get("model_str") or config.get("model_name") + if model_type not in IMF_PRESET_CONFIGS: + raise ValueError(f"Unknown iMF preset '{model_type}'. Known: {list(IMF_PRESET_CONFIGS)}") + + preset = dict(IMF_PRESET_CONFIGS[model_type]) + preset["num_classes"] = int(config.get("num_class_embeds") or config.get("num_classes") or 1000) + preset["model_type"] = model_type + if config.get("sample_size") is not None: + preset["sample_size"] = int(config["sample_size"]) + if config.get("in_channels") is not None: + preset["in_channels"] = int(config["in_channels"]) + return preset + + +def _unsqueeze(t: torch.Tensor, dim: int) -> torch.Tensor: + return t.unsqueeze(dim) + + +class _ScaledLinear(nn.Module): + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + bias_init: str = "zeros", + ): + super().__init__() + self.linear = nn.Linear(in_features, out_features, bias=bias) + if weight_init == "scaled_variance": + std = init_constant / sqrt(in_features) + nn.init.normal_(self.linear.weight, std=std) + elif weight_init == "zeros": + nn.init.zeros_(self.linear.weight) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + if bias: + if bias_init == "zeros": + nn.init.zeros_(self.linear.bias) + else: + raise ValueError(f"Invalid bias_init: {bias_init}") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(x) + + +class _ScaledEmbedding(nn.Module): + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + ): + super().__init__() + self.embedding = nn.Embedding(num_embeddings, embedding_dim) + if weight_init == "scaled_variance": + std = init_constant / sqrt(embedding_dim) + nn.init.normal_(self.embedding.weight, std=std) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.embedding(x) + + +class _RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.eps = eps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + mean_square = torch.mean(torch.square(x), dim=-1, keepdim=True) + output = x * torch.rsqrt(mean_square + self.eps) + return output.to(x.dtype) * self.weight + + +class _SwiGLUMlp(nn.Module): + def __init__( + self, + in_features: int, + hidden_features: int, + weight_init: str = "scaled_variance", + weight_init_constant: float = 1.0, + ): + super().__init__() + init_kwargs = dict(bias=False, weight_init=weight_init, init_constant=weight_init_constant) + self.w1 = _ScaledLinear(in_features, hidden_features, **init_kwargs) + self.w3 = _ScaledLinear(in_features, hidden_features, **init_kwargs) + self.w2 = _ScaledLinear(hidden_features, in_features, **init_kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + + +class IMFTimestepEmbedder(nn.Module): + def __init__( + self, + hidden_size: int, + frequency_embedding_size: int = 256, + init_constant: float = 1.0, + ): + super().__init__() + self.mlp = nn.Sequential( + _ScaledLinear(frequency_embedding_size, hidden_size, bias=True, init_constant=init_constant), + nn.SiLU(), + _ScaledLinear(hidden_size, hidden_size, bias=True, init_constant=init_constant), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32, device=t.device) / half + ) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + embedding = self.timestep_embedding(t, self.frequency_embedding_size) + return self.mlp(embedding.to(dtype=next(self.mlp.parameters()).dtype)) + + +class IMFLabelEmbedder(nn.Module): + def __init__(self, num_classes: int, hidden_size: int, init_constant: float = 1.0): + super().__init__() + self.embedding_table = _ScaledEmbedding(num_classes + 1, hidden_size, init_constant=init_constant) + + def forward(self, labels: torch.Tensor) -> torch.Tensor: + return self.embedding_table(labels) + + +class IMFPatchEmbedder(nn.Module): + def __init__(self, input_size: int, patch_size: int, in_channels: int, hidden_size: int, bias: bool = True): + super().__init__() + self.patch_size = (patch_size, patch_size) + self.num_patches = (input_size // patch_size) ** 2 + self.proj = nn.Conv2d( + in_channels, + hidden_size, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + ) + kh = kw = patch_size + fan_in = kh * kw * in_channels + fan_out = hidden_size + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj.weight, -limit, limit) + if bias: + nn.init.zeros_(self.proj.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj(x) + return x.flatten(2).transpose(1, 2) + + +def precompute_rope_freqs(dim: int, seq_len: int, theta: float = 10000.0, device: torch.device | None = None): + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim)) + positions = torch.arange(seq_len, dtype=torch.float32, device=device) + freqs_cis = torch.outer(positions, freqs) + return torch.complex(torch.cos(freqs_cis), torch.sin(freqs_cis)) + + +def apply_rotary_pos_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + x_ = x.float().reshape(*x.shape[:-1], -1, 2) + x_complex = torch.view_as_complex(x_) + freqs_cis = freqs_cis.unsqueeze(0).unsqueeze(2).to(x.device) + x_rotated = torch.view_as_real(x_complex * freqs_cis).flatten(-2) + return x_rotated.to(dtype=x.dtype) + + +class IMFRoPEAttention(nn.Module): + def __init__(self, hidden_size: int, num_heads: int, weight_init_constant: float = 0.32): + super().__init__() + self.hidden_size = hidden_size + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + init_kwargs = dict(bias=False, init_constant=weight_init_constant) + self.q_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.k_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.v_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.out_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.q_norm = _RMSNorm(self.head_dim) + self.k_norm = _RMSNorm(self.head_dim) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + batch, seq_len, _ = x.shape + q = self.q_proj(x).reshape(batch, seq_len, self.num_heads, self.head_dim) + k = self.k_proj(x).reshape(batch, seq_len, self.num_heads, self.head_dim) + v = self.v_proj(x).reshape(batch, seq_len, self.num_heads, self.head_dim) + q = apply_rotary_pos_emb(self.q_norm(q), rope_freqs) + k = apply_rotary_pos_emb(self.k_norm(k), rope_freqs) + query = q / math.sqrt(self.head_dim) + attn_weights = torch.einsum("bqhd,bkhd->bhqk", query, k) + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(v.dtype) + attn = torch.einsum("bhqk,bkhd->bqhd", attn_weights, v) + return self.out_proj(attn.reshape(batch, seq_len, self.hidden_size)) + + +class IMFTransformerBlock(nn.Module): + def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float, weight_init_constant: float): + super().__init__() + self.norm1 = _RMSNorm(hidden_size) + self.attn = IMFRoPEAttention(hidden_size, num_heads, weight_init_constant=weight_init_constant) + self.norm2 = _RMSNorm(hidden_size) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + self.mlp = _SwiGLUMlp(hidden_size, mlp_hidden_dim, weight_init_constant=weight_init_constant) + self.attn_scale = nn.Parameter(torch.zeros(hidden_size)) + self.mlp_scale = nn.Parameter(torch.zeros(hidden_size)) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), rope_freqs) * self.attn_scale + x = x + self.mlp(self.norm2(x)) * self.mlp_scale + return x + + +class IMFFinalLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int): + super().__init__() + self.norm = _RMSNorm(hidden_size) + self.linear = _ScaledLinear( + hidden_size, + patch_size * patch_size * out_channels, + bias=True, + weight_init="zeros", + init_constant=1.0, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(self.norm(x)) + + +class IMFTransformer2DModel(ModelMixin, ConfigMixin): + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: int = 32, + patch_size: int = 2, + in_channels: int = 4, + hidden_size: int = 768, + depth: int = 12, + num_attention_heads: int = 12, + mlp_ratio: float = 8 / 3, + num_classes: int = 1000, + aux_head_depth: int = 8, + num_class_tokens: int = 8, + num_time_tokens: int = 4, + num_cfg_tokens: int = 4, + num_interval_tokens: int = 2, + token_init_constant: float = 1.0, + embedding_init_constant: float = 1.0, + weight_init_constant: float = 0.32, + eval_mode: bool = True, + model_type: str | None = None, + num_class_embeds: int | None = None, + ): + super().__init__() + if num_class_embeds is not None: + num_classes = int(num_class_embeds) + if model_type in IMF_PRESET_CONFIGS: + preset = IMF_PRESET_CONFIGS[model_type] + sample_size = int(preset["sample_size"]) + patch_size = int(preset["patch_size"]) + hidden_size = int(preset["hidden_size"]) + depth = int(preset["depth"]) + num_attention_heads = int(preset["num_attention_heads"]) + aux_head_depth = int(preset["aux_head_depth"]) + + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.depth = depth + self.num_attention_heads = num_attention_heads + self.num_classes = num_classes + self.aux_head_depth = aux_head_depth + self.num_class_tokens = num_class_tokens + self.num_time_tokens = num_time_tokens + self.num_cfg_tokens = num_cfg_tokens + self.num_interval_tokens = num_interval_tokens + self.eval_mode = eval_mode + self.gradient_checkpointing = False + + self.x_embedder = IMFPatchEmbedder(sample_size, patch_size, in_channels, hidden_size, bias=True) + embed_kwargs = dict(hidden_size=hidden_size, init_constant=embedding_init_constant) + self.h_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.omega_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_start_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_end_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.y_embedder = IMFLabelEmbedder(num_classes, hidden_size, init_constant=embedding_init_constant) + + token_std = token_init_constant / math.sqrt(hidden_size) + self.time_tokens = nn.Parameter(torch.randn(num_time_tokens, hidden_size) * token_std) + self.class_tokens = nn.Parameter(torch.randn(num_class_tokens, hidden_size) * token_std) + self.omega_tokens = nn.Parameter(torch.randn(num_cfg_tokens, hidden_size) * token_std) + self.t_min_tokens = nn.Parameter(torch.randn(num_interval_tokens, hidden_size) * token_std) + self.t_max_tokens = nn.Parameter(torch.randn(num_interval_tokens, hidden_size) * token_std) + + total_tokens = ( + self.x_embedder.num_patches + + num_class_tokens + + num_cfg_tokens + + 2 * num_interval_tokens + + num_time_tokens + ) + self.prefix_tokens = num_class_tokens + num_cfg_tokens + 2 * num_interval_tokens + num_time_tokens + self.head_dim = hidden_size // num_attention_heads + self.register_buffer( + "rope_freqs", + precompute_rope_freqs(self.head_dim, total_tokens), + persistent=False, + ) + + shared_depth = depth - aux_head_depth + block_kwargs = dict( + hidden_size=hidden_size, + num_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + weight_init_constant=weight_init_constant, + ) + self.shared_blocks = nn.ModuleList([IMFTransformerBlock(**block_kwargs) for _ in range(shared_depth)]) + self.u_heads = nn.ModuleList([IMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth)]) + self.v_heads = nn.ModuleList( + [IMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth if not eval_mode else 0)] + ) + self.u_final_layer = IMFFinalLayer(hidden_size, patch_size, in_channels) + self.v_final_layer = IMFFinalLayer(hidden_size, patch_size, in_channels) + + def unpatchify(self, x: torch.Tensor) -> torch.Tensor: + c = self.out_channels + p = self.patch_size + h = w = int(x.shape[1] ** 0.5) + x = x.reshape(x.shape[0], h, w, p, p, c) + x = torch.einsum("nhwpqc->nchpwq", x) + return x.reshape(x.shape[0], c, h * p, w * p) + + def _build_sequence( + self, + sample: torch.Tensor, + time_gap: torch.Tensor, + guidance_scale: torch.Tensor, + guidance_interval_start: torch.Tensor, + guidance_interval_end: torch.Tensor, + class_labels: torch.Tensor, + ) -> torch.Tensor: + x_embed = self.x_embedder(sample) + h_embed = self.h_embedder(time_gap) + omega_embed = self.omega_embedder(1 - 1 / guidance_scale) + t_min_embed = self.cfg_t_start_embedder(guidance_interval_start) + t_max_embed = self.cfg_t_end_embedder(guidance_interval_end) + y_embed = self.y_embedder(class_labels) + + time_tokens = self.time_tokens + _unsqueeze(h_embed, 1) + omega_tokens = self.omega_tokens + _unsqueeze(omega_embed, 1) + t_min_tokens = self.t_min_tokens + _unsqueeze(t_min_embed, 1) + t_max_tokens = self.t_max_tokens + _unsqueeze(t_max_embed, 1) + class_tokens = self.class_tokens + _unsqueeze(y_embed, 1) + + return torch.cat( + [class_tokens, omega_tokens, t_min_tokens, t_max_tokens, time_tokens, x_embed], + dim=1, + ) + + def forward( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + time_gap: Optional[torch.Tensor] = None, + guidance_scale: Optional[torch.Tensor] = None, + guidance_interval_start: Optional[torch.Tensor] = None, + guidance_interval_end: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[IMFTransformer2DModelOutput, Tuple[torch.Tensor, ...]]: + batch_size = sample.shape[0] + timestep = self._expand_batch(timestep, batch_size, sample.device, sample.dtype) + if time_gap is None: + time_gap = timestep + else: + time_gap = self._expand_batch(time_gap, batch_size, sample.device, sample.dtype) + if guidance_scale is None: + guidance_scale = torch.ones(batch_size, device=sample.device, dtype=sample.dtype) + else: + guidance_scale = self._expand_batch(guidance_scale, batch_size, sample.device, sample.dtype) + if guidance_interval_start is None: + guidance_interval_start = torch.zeros(batch_size, device=sample.device, dtype=sample.dtype) + else: + guidance_interval_start = self._expand_batch( + guidance_interval_start, batch_size, sample.device, sample.dtype + ) + if guidance_interval_end is None: + guidance_interval_end = torch.ones(batch_size, device=sample.device, dtype=sample.dtype) + else: + guidance_interval_end = self._expand_batch( + guidance_interval_end, batch_size, sample.device, sample.dtype + ) + + class_labels = class_labels.reshape(-1).long() + seq = self._build_sequence( + sample, + time_gap, + guidance_scale, + guidance_interval_start, + guidance_interval_end, + class_labels, + ) + + for block in self.shared_blocks: + if self.training and self.gradient_checkpointing: + seq = torch.utils.checkpoint.checkpoint(block, seq, self.rope_freqs, use_reentrant=False) + else: + seq = block(seq, self.rope_freqs) + + u_seq = v_seq = seq + for block in self.u_heads: + u_seq = block(u_seq, self.rope_freqs) + for block in self.v_heads: + v_seq = block(v_seq, self.rope_freqs) + + u_tokens = u_seq[:, self.prefix_tokens :] + velocity_u = self.unpatchify(self.u_final_layer(u_tokens)) + + velocity_v = None + if len(self.v_heads) > 0: + v_tokens = v_seq[:, self.prefix_tokens :] + velocity_v = self.unpatchify(self.v_final_layer(v_tokens)) + + if not return_dict: + if velocity_v is None: + return (velocity_u,) + return (velocity_u, velocity_v) + return IMFTransformer2DModelOutput(velocity_u=velocity_u, velocity_v=velocity_v) + + @staticmethod + def _expand_batch( + value: torch.Tensor, + batch_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + value = torch.as_tensor(value, device=device, dtype=dtype) + if value.ndim == 0: + value = value.reshape(1).repeat(batch_size) + else: + value = value.reshape(-1) + if value.shape[0] == 1 and batch_size > 1: + value = value.repeat(batch_size) + return value + + @classmethod + def from_imf_checkpoint( + cls, + checkpoint_path: str, + model_type: str | None = None, + map_location: str = "cpu", + strict: bool = True, + eval_mode: bool = True, + ) -> Tuple["IMFTransformer2DModel", Dict[str, object]]: + checkpoint = torch.load(checkpoint_path, map_location=map_location, weights_only=False) + if isinstance(checkpoint, Mapping) and "state_dict" in checkpoint: + state_dict = checkpoint["state_dict"] + elif isinstance(checkpoint, Mapping) and any(k.startswith("net.") for k in checkpoint): + state_dict = checkpoint + else: + state_dict = checkpoint + + inferred_type = model_type + if inferred_type is None and isinstance(checkpoint, Mapping): + args = checkpoint.get("args") + if isinstance(args, argparse.Namespace): + inferred_type = getattr(args, "model", None) or getattr(args, "model_str", None) + elif isinstance(args, Mapping): + inferred_type = args.get("model") or args.get("model_str") + + if inferred_type is None: + raise ValueError("model_type must be provided when loading a raw state dict checkpoint.") + + config = dict(IMF_PRESET_CONFIGS[inferred_type]) + config["model_type"] = inferred_type + config["eval_mode"] = eval_mode + model = cls(**config) + model.load_state_dict(remap_legacy_state_dict(state_dict), strict=strict) + metadata = {"checkpoint_path": checkpoint_path, "model_type": inferred_type} + return model, metadata + + def to_imf_checkpoint(self, prefix: str = "net.") -> Dict[str, torch.Tensor]: + state: Dict[str, torch.Tensor] = {} + for key, value in self.state_dict().items(): + if key == "rope_freqs": + continue + state[f"{prefix}{key}"] = value.detach().cpu() + return state + + @property + def net(self): + return self + + +IMFDiffusersModel = IMFTransformer2DModel diff --git a/iMF-L-SIM/vae/config.json b/iMF-L-SIM/vae/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ae2ab37de2acb8bc348d3de809ae8385324be962 --- /dev/null +++ b/iMF-L-SIM/vae/config.json @@ -0,0 +1,38 @@ +{ + "_class_name": "AutoencoderKL", + "_diffusers_version": "0.36.0", + "_name_or_path": "stabilityai/sd-vae-ft-mse", + "act_fn": "silu", + "block_out_channels": [ + 128, + 256, + 512, + 512 + ], + "down_block_types": [ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D" + ], + "force_upcast": true, + "in_channels": 3, + "latent_channels": 4, + "latents_mean": null, + "latents_std": null, + "layers_per_block": 2, + "mid_block_add_attention": true, + "norm_num_groups": 32, + "out_channels": 3, + "sample_size": 256, + "scaling_factor": 0.18215, + "shift_factor": null, + "up_block_types": [ + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D" + ], + "use_post_quant_conv": true, + "use_quant_conv": true +} diff --git a/iMF-L-SIM/vae/diffusion_pytorch_model.safetensors b/iMF-L-SIM/vae/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..90464d67ac7303d0ee4696334df13da130a948ea --- /dev/null +++ b/iMF-L-SIM/vae/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d993488569e928462932c8c38a0760b874d166399b14414135bd9c42df5815 +size 334643276 diff --git a/iMF-XL-SIM/README.md b/iMF-XL-SIM/README.md new file mode 100644 index 0000000000000000000000000000000000000000..431ad8f06d3f89577b1ac01b21e4bfd0fb272f52 --- /dev/null +++ b/iMF-XL-SIM/README.md @@ -0,0 +1,54 @@ +--- +license: mit +library_name: diffusers +pipeline_tag: text-to-image +tags: +- diffusers +- imf +- image-generation +- class-conditional +inference: true +widget: +- output: + url: demo.png +language: +- en +--- + +# iMF-XL-SIM + +Self-contained Diffusers variant for **iMF-XL/2 (FD-SIM post-trained)** (Improved Mean Flows). + +## Demo + +![iMF-XL-SIM demo](demo.png) + +Class-conditional sample (ImageNet class **207**, golden retriever), 1 step, CFG 1.8, interval [0.0, 1.0], seed 42. + +## Load + +```python +from pathlib import Path +from diffusers import DiffusionPipeline +import torch + +model_dir = Path("iMF-XL-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + local_files_only=True, + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.bfloat16, +).to("cuda") + +generator = torch.Generator(device="cuda").manual_seed(42) +image = pipe( + class_labels="golden retriever", + num_inference_steps=1, + guidance_scale=1.8, + guidance_interval_start=0.0, + guidance_interval_end=1.0, + generator=generator, +).images[0] +image.save("demo.png") +``` diff --git a/iMF-XL-SIM/model_index.json b/iMF-XL-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..80209bf0b9dd1de11a6a86528d2ebb23e3fe20c1 --- /dev/null +++ b/iMF-XL-SIM/model_index.json @@ -0,0 +1,1021 @@ +{ + "_class_name": [ + "pipeline", + "IMFPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_imf", + "IMFScheduler" + ], + "transformer": [ + "transformer_imf", + "IMFTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + }, + "vae": [ + "diffusers", + "AutoencoderKL" + ] +} diff --git a/iMF-XL-SIM/pipeline.py b/iMF-XL-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..f137dbd473c0740293beff2648a0f78bf823ecce --- /dev/null +++ b/iMF-XL-SIM/pipeline.py @@ -0,0 +1,342 @@ +"""Hub custom pipeline: IMFPipeline. +Load with native Hugging Face diffusers and trust_remote_code=True. +""" + +from __future__ import annotations + +import importlib.util +import inspect + +import json +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union, Any + +import torch +from diffusers.image_processor import VaeImageProcessor +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +# imeanflow / FD-Loss sd-vae latent statistics (stabilityai/sd-vae-ft-mse) +LATENT_CHANNEL_MEAN = (0.86488, -0.27787343, 0.21616915, 0.3738409) +LATENT_CHANNEL_STD = (4.85503674, 5.31922414, 3.93725398, 3.9870003) + + +def _load_bundled_vae(transformer) -> Optional[Any]: + transformer_path = getattr(transformer.config, "_name_or_path", None) + if not transformer_path: + return None + vae_dir = Path(transformer_path).resolve().parent / "vae" + if not vae_dir.is_dir() or not (vae_dir / "config.json").is_file(): + return None + from diffusers import AutoencoderKL + + vae_dtype = getattr(transformer, "dtype", torch.float32) + return AutoencoderKL.from_pretrained(str(vae_dir), torch_dtype=vae_dtype) + + +class IMFPipeline(DiffusionPipeline): + + @staticmethod + def prepare_extra_step_kwargs( + scheduler, + generator=None, + eta: float | None = None, + ): + kwargs = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _prepare_generator( + generator: Optional[Union[torch.Generator, List[torch.Generator]]], + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(generator, list): + for gen in generator: + if gen is not None: + gen.manual_seed(int(gen.initial_seed())) + return generator + generator.manual_seed(int(generator.initial_seed())) + return generator + + @staticmethod + def _coerce_scheduler(scheduler, transformer) -> Any: + if scheduler is not None and not isinstance(scheduler, (list, tuple)): + return scheduler + variant_path = getattr(transformer.config, "_name_or_path", None) + if variant_path: + scheduler_dir = Path(variant_path).resolve().parent / "scheduler" + module_path = scheduler_dir / "scheduling_imf.py" + config_path = scheduler_dir / "scheduler_config.json" + if module_path.is_file() and config_path.is_file(): + spec = importlib.util.spec_from_file_location("scheduling_imf", module_path) + if spec is not None and spec.loader is not None: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.IMFScheduler.from_pretrained(str(scheduler_dir)) + raise ValueError( + "IMFPipeline could not load IMFScheduler. Ensure the variant includes scheduler/scheduling_imf.py." + ) + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + model_cpu_offload_seq = "transformer->vae" + + def __init__( + self, + transformer, + scheduler, + vae=None, + id2label: Optional[Dict[Union[int, str], str]] = None, + ): + super().__init__() + scheduler = self._coerce_scheduler(scheduler, transformer) + if scheduler is None: + raise ValueError("IMFPipeline requires a scheduler loaded from the checkpoint.") + if isinstance(vae, (list, tuple)): + vae = None + if vae is None: + vae = _load_bundled_vae(transformer) + if vae is not None: + vae = vae.to(device=transformer.device, dtype=getattr(transformer, "dtype", torch.float32)) + self.register_modules(transformer=transformer, scheduler=scheduler, vae=vae) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + self.latent_channel_mean = torch.tensor(LATENT_CHANNEL_MEAN, dtype=torch.float32).view(1, 4, 1, 1) + self.latent_channel_std = torch.tensor(LATENT_CHANNEL_STD, dtype=torch.float32).view(1, 4, 1, 1) + vae_scale_factor = 8 + if vae is not None: + vae_scale_factor = 2 ** (len(vae.config.block_out_channels) - 1) + self.vae_scale_factor = vae_scale_factor + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + loaded = self._read_id2label_from_model_index(getattr(self.config, "_name_or_path", None)) + if loaded: + self._id2label = loaded + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _read_id2label_from_model_index(variant_path: Optional[str]) -> Dict[int, str]: + if not variant_path: + return {} + model_index_path = Path(variant_path).resolve() / "model_index.json" + if not model_index_path.exists(): + return {} + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if not isinstance(id2label, dict): + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + self._ensure_labels_loaded() + if not self.labels: + raise ValueError("No labels loaded. Ensure `id2label` exists in model_index.json.") + if isinstance(label, str): + label = [label] + missing = [item for item in label if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Example valid labels: {preview}, ...") + return [self.labels[item] for item in label] + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) + return list(class_labels) + + def _denormalize_latents(self, latents: torch.Tensor) -> torch.Tensor: + mean = self.latent_channel_mean.to(device=latents.device, dtype=latents.dtype) + std = self.latent_channel_std.to(device=latents.device, dtype=latents.dtype) + return latents * std + mean + + def decode_latents(self, latents: torch.Tensor, output_type: str = "pil"): + if output_type == "latent": + return latents + if self.vae is None: + raise ValueError( + "Cannot decode latents without a VAE. Ensure model_index.json lists vae and the variant includes vae/." + ) + vae_dtype = next(self.vae.parameters()).dtype + if next(self.vae.parameters()).device != latents.device: + self.vae.to(device=latents.device, dtype=vae_dtype) + latents = self._denormalize_latents(latents).to(dtype=vae_dtype) + image = self.vae.decode(latents).sample + return self.image_processor.postprocess(image, output_type=output_type) + + def _predict_velocity_u( + self, + latents: torch.Tensor, + timestep: torch.Tensor, + time_gap: torch.Tensor, + class_labels: torch.Tensor, + class_null: torch.Tensor, + guidance_scale: float, + guidance_interval_start: float, + guidance_interval_end: float, + do_classifier_free_guidance: bool, + ) -> torch.Tensor: + if do_classifier_free_guidance: + latents_in = torch.cat([latents, latents], dim=0) + labels = torch.cat([class_labels, class_null], dim=0) + omega = torch.tensor([guidance_scale, 1.0], device=latents.device, dtype=latents.dtype) + t_min = torch.tensor([guidance_interval_start, 0.0], device=latents.device, dtype=latents.dtype) + t_max = torch.tensor([guidance_interval_end, 1.0], device=latents.device, dtype=latents.dtype) + batch = latents.shape[0] + timestep_in = timestep.reshape(1).repeat(2 * batch).to(dtype=latents.dtype) + time_gap_in = time_gap.reshape(1).repeat(2 * batch).to(dtype=latents.dtype) + omega = omega.repeat(batch) + t_min = t_min.repeat(batch) + t_max = t_max.repeat(batch) + else: + latents_in = latents + labels = class_labels + batch = latents.shape[0] + timestep_in = timestep.reshape(1).repeat(batch).to(dtype=latents.dtype) + time_gap_in = time_gap.reshape(1).repeat(batch).to(dtype=latents.dtype) + omega = torch.full((batch,), guidance_scale, device=latents.device, dtype=latents.dtype) + t_min = torch.full((batch,), guidance_interval_start, device=latents.device, dtype=latents.dtype) + t_max = torch.full((batch,), guidance_interval_end, device=latents.device, dtype=latents.dtype) + + outputs = self.transformer( + sample=latents_in, + timestep=timestep_in, + class_labels=labels, + time_gap=time_gap_in, + guidance_scale=omega, + guidance_interval_start=t_min, + guidance_interval_end=t_max, + return_dict=True, + ) + velocity_u = outputs.velocity_u + + if not do_classifier_free_guidance: + return velocity_u + + u_cond, u_uncond = velocity_u.chunk(2, dim=0) + return u_uncond + guidance_scale * (u_cond - u_uncond) + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + num_inference_steps: int = 1, + guidance_scale: float = 2.7, + guidance_interval_start: float = 0.1, + guidance_interval_end: float = 0.9, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + if output_type not in {"pil", "np", "pt", "latent"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt', 'latent'.") + + class_label_ids = self._normalize_class_labels(class_labels) + do_classifier_free_guidance = guidance_scale > 1.0 + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + + image_size = int(self.transformer.config.sample_size) + channels = int(self.transformer.config.in_channels) + null_class_val = int(getattr(self.transformer.config, "num_classes", 1000)) + + generator = self._prepare_generator(generator) + + if latents is None: + latents = randn_tensor( + shape=(batch_size, channels, image_size, image_size), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) + + class_labels_t = torch.tensor(class_label_ids, device=latents.device, dtype=torch.long).reshape(-1) + class_labels_t = class_labels_t.clamp(0, null_class_val - 1) + class_null = torch.full_like(class_labels_t, null_class_val) + + self.scheduler.set_timesteps(num_inference_steps, device=latents.device) + timesteps = self.scheduler.timesteps + + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + + for i in self.progress_bar(range(num_inference_steps)): + t = timesteps[i] + t_next = timesteps[i + 1] + time_gap = t - t_next + velocity_u = self._predict_velocity_u( + latents, + t, + time_gap, + class_labels_t, + class_null, + guidance_scale, + guidance_interval_start, + guidance_interval_end, + do_classifier_free_guidance, + ) + latents = self.scheduler.step(velocity_u, t, latents, **extra_step_kwargs).prev_sample + + images = self.decode_latents(latents, output_type=output_type) + + self.maybe_free_model_hooks() + + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) + + +IMFPipelineOutput = ImagePipelineOutput diff --git a/iMF-XL-SIM/scheduler/scheduler_config.json b/iMF-XL-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c643eeffcc65000209b57865d0f68d2870d004a8 --- /dev/null +++ b/iMF-XL-SIM/scheduler/scheduler_config.json @@ -0,0 +1,5 @@ +{ + "_class_name": "IMFScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000 +} diff --git a/iMF-XL-SIM/scheduler/scheduling_imf.py b/iMF-XL-SIM/scheduler/scheduling_imf.py new file mode 100644 index 0000000000000000000000000000000000000000..7dfaeeb301cf121a4ec6a24e4cf6ab5fe33fae6e --- /dev/null +++ b/iMF-XL-SIM/scheduler/scheduling_imf.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class IMFSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class IMFScheduler(SchedulerMixin, ConfigMixin): + """Mean-flow scheduler with timesteps from 1.0 to 0.0.""" + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + shift: float = 1.0, + stochastic_sampling: bool = False, + ): + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> None: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.flatten()[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[IMFSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = (t - t_next).to(dtype=sample.dtype, device=sample.device) + while dt.ndim < sample.ndim: + dt = dt.unsqueeze(-1) + + if getattr(self.config, "stochastic_sampling", False): + t_bc = t.to(dtype=sample.dtype, device=sample.device) + while t_bc.ndim < sample.ndim: + t_bc = t_bc.unsqueeze(-1) + t_next_bc = t_next.to(dtype=sample.dtype, device=sample.device) + while t_next_bc.ndim < sample.ndim: + t_next_bc = t_next_bc.unsqueeze(-1) + x0 = sample - t_bc * model_output + noise = randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + prev_sample = (1.0 - t_next_bc) * x0 + t_next_bc * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return IMFSchedulerOutput(prev_sample=prev_sample) diff --git a/iMF-XL-SIM/transformer/config.json b/iMF-XL-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..45eb6c32790545080e829b8427b704e8376f463b --- /dev/null +++ b/iMF-XL-SIM/transformer/config.json @@ -0,0 +1,23 @@ +{ + "_class_name": "IMFTransformer2DModel", + "_diffusers_version": "0.38.0", + "aux_head_depth": 8, + "depth": 48, + "embedding_init_constant": 1.0, + "eval_mode": true, + "hidden_size": 1024, + "in_channels": 4, + "mlp_ratio": 2.6666666666666665, + "model_type": "iMF-XL/2", + "num_attention_heads": 16, + "num_cfg_tokens": 4, + "num_class_embeds": null, + "num_class_tokens": 8, + "num_classes": 1000, + "num_interval_tokens": 2, + "num_time_tokens": 4, + "patch_size": 2, + "sample_size": 32, + "token_init_constant": 1.0, + "weight_init_constant": 0.32 +} diff --git a/iMF-XL-SIM/transformer/diffusion_pytorch_model.safetensors b/iMF-XL-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4625fd773735a9a6e4ec71c3440b276501b8b976 --- /dev/null +++ b/iMF-XL-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f741e0f20eec57fa030c09939973eccc21316d97f8d403bc9dcd9ef95915c18 +size 2441803104 diff --git a/iMF-XL-SIM/transformer/transformer_imf.py b/iMF-XL-SIM/transformer/transformer_imf.py new file mode 100644 index 0000000000000000000000000000000000000000..be1783e9bd8ae5b81a82c8d01640e0d214004c26 --- /dev/null +++ b/iMF-XL-SIM/transformer/transformer_imf.py @@ -0,0 +1,586 @@ +from __future__ import annotations + +import argparse +import math +from collections.abc import Mapping +from dataclasses import dataclass +from functools import partial +from math import sqrt +from typing import Dict, Literal, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils import BaseOutput + + +IMF_PRESET_CONFIGS: Dict[str, Dict[str, object]] = { + "iMF-B/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 768, + "depth": 12, + "num_attention_heads": 12, + "aux_head_depth": 8, + }, + "iMF-M/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 768, + "depth": 24, + "num_attention_heads": 12, + "aux_head_depth": 8, + }, + "iMF-L/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "aux_head_depth": 8, + }, + "iMF-XL/2": { + "sample_size": 32, + "patch_size": 2, + "hidden_size": 1024, + "depth": 48, + "num_attention_heads": 16, + "aux_head_depth": 8, + }, +} + + +@dataclass +class IMFTransformer2DModelOutput(BaseOutput): + velocity_u: torch.Tensor + velocity_v: Optional[torch.Tensor] = None + + +def remap_legacy_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Map legacy iMF / Flax-style keys to native IMFTransformer2DModel keys.""" + remapped: Dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + for prefix in ("net.", "transformer."): + if new_key.startswith(prefix): + new_key = new_key[len(prefix) :] + break + new_key = new_key.replace(".kernel", ".weight") + new_key = new_key.replace("._flax_linear.", ".linear.") + new_key = new_key.replace("._flax_embedding.", ".embedding.") + remapped[new_key] = value + return remapped + + +def config_from_legacy(config: Dict[str, object]) -> Dict[str, object]: + model_type = config.get("model_type") or config.get("model_str") or config.get("model_name") + if model_type not in IMF_PRESET_CONFIGS: + raise ValueError(f"Unknown iMF preset '{model_type}'. Known: {list(IMF_PRESET_CONFIGS)}") + + preset = dict(IMF_PRESET_CONFIGS[model_type]) + preset["num_classes"] = int(config.get("num_class_embeds") or config.get("num_classes") or 1000) + preset["model_type"] = model_type + if config.get("sample_size") is not None: + preset["sample_size"] = int(config["sample_size"]) + if config.get("in_channels") is not None: + preset["in_channels"] = int(config["in_channels"]) + return preset + + +def _unsqueeze(t: torch.Tensor, dim: int) -> torch.Tensor: + return t.unsqueeze(dim) + + +class _ScaledLinear(nn.Module): + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + bias_init: str = "zeros", + ): + super().__init__() + self.linear = nn.Linear(in_features, out_features, bias=bias) + if weight_init == "scaled_variance": + std = init_constant / sqrt(in_features) + nn.init.normal_(self.linear.weight, std=std) + elif weight_init == "zeros": + nn.init.zeros_(self.linear.weight) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + if bias: + if bias_init == "zeros": + nn.init.zeros_(self.linear.bias) + else: + raise ValueError(f"Invalid bias_init: {bias_init}") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(x) + + +class _ScaledEmbedding(nn.Module): + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + ): + super().__init__() + self.embedding = nn.Embedding(num_embeddings, embedding_dim) + if weight_init == "scaled_variance": + std = init_constant / sqrt(embedding_dim) + nn.init.normal_(self.embedding.weight, std=std) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.embedding(x) + + +class _RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.eps = eps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + mean_square = torch.mean(torch.square(x), dim=-1, keepdim=True) + output = x * torch.rsqrt(mean_square + self.eps) + return output.to(x.dtype) * self.weight + + +class _SwiGLUMlp(nn.Module): + def __init__( + self, + in_features: int, + hidden_features: int, + weight_init: str = "scaled_variance", + weight_init_constant: float = 1.0, + ): + super().__init__() + init_kwargs = dict(bias=False, weight_init=weight_init, init_constant=weight_init_constant) + self.w1 = _ScaledLinear(in_features, hidden_features, **init_kwargs) + self.w3 = _ScaledLinear(in_features, hidden_features, **init_kwargs) + self.w2 = _ScaledLinear(hidden_features, in_features, **init_kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + + +class IMFTimestepEmbedder(nn.Module): + def __init__( + self, + hidden_size: int, + frequency_embedding_size: int = 256, + init_constant: float = 1.0, + ): + super().__init__() + self.mlp = nn.Sequential( + _ScaledLinear(frequency_embedding_size, hidden_size, bias=True, init_constant=init_constant), + nn.SiLU(), + _ScaledLinear(hidden_size, hidden_size, bias=True, init_constant=init_constant), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32, device=t.device) / half + ) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + embedding = self.timestep_embedding(t, self.frequency_embedding_size) + return self.mlp(embedding.to(dtype=next(self.mlp.parameters()).dtype)) + + +class IMFLabelEmbedder(nn.Module): + def __init__(self, num_classes: int, hidden_size: int, init_constant: float = 1.0): + super().__init__() + self.embedding_table = _ScaledEmbedding(num_classes + 1, hidden_size, init_constant=init_constant) + + def forward(self, labels: torch.Tensor) -> torch.Tensor: + return self.embedding_table(labels) + + +class IMFPatchEmbedder(nn.Module): + def __init__(self, input_size: int, patch_size: int, in_channels: int, hidden_size: int, bias: bool = True): + super().__init__() + self.patch_size = (patch_size, patch_size) + self.num_patches = (input_size // patch_size) ** 2 + self.proj = nn.Conv2d( + in_channels, + hidden_size, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + ) + kh = kw = patch_size + fan_in = kh * kw * in_channels + fan_out = hidden_size + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj.weight, -limit, limit) + if bias: + nn.init.zeros_(self.proj.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj(x) + return x.flatten(2).transpose(1, 2) + + +def precompute_rope_freqs(dim: int, seq_len: int, theta: float = 10000.0, device: torch.device | None = None): + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim)) + positions = torch.arange(seq_len, dtype=torch.float32, device=device) + freqs_cis = torch.outer(positions, freqs) + return torch.complex(torch.cos(freqs_cis), torch.sin(freqs_cis)) + + +def apply_rotary_pos_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + x_ = x.float().reshape(*x.shape[:-1], -1, 2) + x_complex = torch.view_as_complex(x_) + freqs_cis = freqs_cis.unsqueeze(0).unsqueeze(2).to(x.device) + x_rotated = torch.view_as_real(x_complex * freqs_cis).flatten(-2) + return x_rotated.to(dtype=x.dtype) + + +class IMFRoPEAttention(nn.Module): + def __init__(self, hidden_size: int, num_heads: int, weight_init_constant: float = 0.32): + super().__init__() + self.hidden_size = hidden_size + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + init_kwargs = dict(bias=False, init_constant=weight_init_constant) + self.q_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.k_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.v_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.out_proj = _ScaledLinear(hidden_size, hidden_size, **init_kwargs) + self.q_norm = _RMSNorm(self.head_dim) + self.k_norm = _RMSNorm(self.head_dim) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + batch, seq_len, _ = x.shape + q = self.q_proj(x).reshape(batch, seq_len, self.num_heads, self.head_dim) + k = self.k_proj(x).reshape(batch, seq_len, self.num_heads, self.head_dim) + v = self.v_proj(x).reshape(batch, seq_len, self.num_heads, self.head_dim) + q = apply_rotary_pos_emb(self.q_norm(q), rope_freqs) + k = apply_rotary_pos_emb(self.k_norm(k), rope_freqs) + query = q / math.sqrt(self.head_dim) + attn_weights = torch.einsum("bqhd,bkhd->bhqk", query, k) + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(v.dtype) + attn = torch.einsum("bhqk,bkhd->bqhd", attn_weights, v) + return self.out_proj(attn.reshape(batch, seq_len, self.hidden_size)) + + +class IMFTransformerBlock(nn.Module): + def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float, weight_init_constant: float): + super().__init__() + self.norm1 = _RMSNorm(hidden_size) + self.attn = IMFRoPEAttention(hidden_size, num_heads, weight_init_constant=weight_init_constant) + self.norm2 = _RMSNorm(hidden_size) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + self.mlp = _SwiGLUMlp(hidden_size, mlp_hidden_dim, weight_init_constant=weight_init_constant) + self.attn_scale = nn.Parameter(torch.zeros(hidden_size)) + self.mlp_scale = nn.Parameter(torch.zeros(hidden_size)) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), rope_freqs) * self.attn_scale + x = x + self.mlp(self.norm2(x)) * self.mlp_scale + return x + + +class IMFFinalLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int): + super().__init__() + self.norm = _RMSNorm(hidden_size) + self.linear = _ScaledLinear( + hidden_size, + patch_size * patch_size * out_channels, + bias=True, + weight_init="zeros", + init_constant=1.0, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(self.norm(x)) + + +class IMFTransformer2DModel(ModelMixin, ConfigMixin): + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: int = 32, + patch_size: int = 2, + in_channels: int = 4, + hidden_size: int = 768, + depth: int = 12, + num_attention_heads: int = 12, + mlp_ratio: float = 8 / 3, + num_classes: int = 1000, + aux_head_depth: int = 8, + num_class_tokens: int = 8, + num_time_tokens: int = 4, + num_cfg_tokens: int = 4, + num_interval_tokens: int = 2, + token_init_constant: float = 1.0, + embedding_init_constant: float = 1.0, + weight_init_constant: float = 0.32, + eval_mode: bool = True, + model_type: str | None = None, + num_class_embeds: int | None = None, + ): + super().__init__() + if num_class_embeds is not None: + num_classes = int(num_class_embeds) + if model_type in IMF_PRESET_CONFIGS: + preset = IMF_PRESET_CONFIGS[model_type] + sample_size = int(preset["sample_size"]) + patch_size = int(preset["patch_size"]) + hidden_size = int(preset["hidden_size"]) + depth = int(preset["depth"]) + num_attention_heads = int(preset["num_attention_heads"]) + aux_head_depth = int(preset["aux_head_depth"]) + + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.depth = depth + self.num_attention_heads = num_attention_heads + self.num_classes = num_classes + self.aux_head_depth = aux_head_depth + self.num_class_tokens = num_class_tokens + self.num_time_tokens = num_time_tokens + self.num_cfg_tokens = num_cfg_tokens + self.num_interval_tokens = num_interval_tokens + self.eval_mode = eval_mode + self.gradient_checkpointing = False + + self.x_embedder = IMFPatchEmbedder(sample_size, patch_size, in_channels, hidden_size, bias=True) + embed_kwargs = dict(hidden_size=hidden_size, init_constant=embedding_init_constant) + self.h_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.omega_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_start_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_end_embedder = IMFTimestepEmbedder(**embed_kwargs) + self.y_embedder = IMFLabelEmbedder(num_classes, hidden_size, init_constant=embedding_init_constant) + + token_std = token_init_constant / math.sqrt(hidden_size) + self.time_tokens = nn.Parameter(torch.randn(num_time_tokens, hidden_size) * token_std) + self.class_tokens = nn.Parameter(torch.randn(num_class_tokens, hidden_size) * token_std) + self.omega_tokens = nn.Parameter(torch.randn(num_cfg_tokens, hidden_size) * token_std) + self.t_min_tokens = nn.Parameter(torch.randn(num_interval_tokens, hidden_size) * token_std) + self.t_max_tokens = nn.Parameter(torch.randn(num_interval_tokens, hidden_size) * token_std) + + total_tokens = ( + self.x_embedder.num_patches + + num_class_tokens + + num_cfg_tokens + + 2 * num_interval_tokens + + num_time_tokens + ) + self.prefix_tokens = num_class_tokens + num_cfg_tokens + 2 * num_interval_tokens + num_time_tokens + self.head_dim = hidden_size // num_attention_heads + self.register_buffer( + "rope_freqs", + precompute_rope_freqs(self.head_dim, total_tokens), + persistent=False, + ) + + shared_depth = depth - aux_head_depth + block_kwargs = dict( + hidden_size=hidden_size, + num_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + weight_init_constant=weight_init_constant, + ) + self.shared_blocks = nn.ModuleList([IMFTransformerBlock(**block_kwargs) for _ in range(shared_depth)]) + self.u_heads = nn.ModuleList([IMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth)]) + self.v_heads = nn.ModuleList( + [IMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth if not eval_mode else 0)] + ) + self.u_final_layer = IMFFinalLayer(hidden_size, patch_size, in_channels) + self.v_final_layer = IMFFinalLayer(hidden_size, patch_size, in_channels) + + def unpatchify(self, x: torch.Tensor) -> torch.Tensor: + c = self.out_channels + p = self.patch_size + h = w = int(x.shape[1] ** 0.5) + x = x.reshape(x.shape[0], h, w, p, p, c) + x = torch.einsum("nhwpqc->nchpwq", x) + return x.reshape(x.shape[0], c, h * p, w * p) + + def _build_sequence( + self, + sample: torch.Tensor, + time_gap: torch.Tensor, + guidance_scale: torch.Tensor, + guidance_interval_start: torch.Tensor, + guidance_interval_end: torch.Tensor, + class_labels: torch.Tensor, + ) -> torch.Tensor: + x_embed = self.x_embedder(sample) + h_embed = self.h_embedder(time_gap) + omega_embed = self.omega_embedder(1 - 1 / guidance_scale) + t_min_embed = self.cfg_t_start_embedder(guidance_interval_start) + t_max_embed = self.cfg_t_end_embedder(guidance_interval_end) + y_embed = self.y_embedder(class_labels) + + time_tokens = self.time_tokens + _unsqueeze(h_embed, 1) + omega_tokens = self.omega_tokens + _unsqueeze(omega_embed, 1) + t_min_tokens = self.t_min_tokens + _unsqueeze(t_min_embed, 1) + t_max_tokens = self.t_max_tokens + _unsqueeze(t_max_embed, 1) + class_tokens = self.class_tokens + _unsqueeze(y_embed, 1) + + return torch.cat( + [class_tokens, omega_tokens, t_min_tokens, t_max_tokens, time_tokens, x_embed], + dim=1, + ) + + def forward( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + time_gap: Optional[torch.Tensor] = None, + guidance_scale: Optional[torch.Tensor] = None, + guidance_interval_start: Optional[torch.Tensor] = None, + guidance_interval_end: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[IMFTransformer2DModelOutput, Tuple[torch.Tensor, ...]]: + batch_size = sample.shape[0] + timestep = self._expand_batch(timestep, batch_size, sample.device, sample.dtype) + if time_gap is None: + time_gap = timestep + else: + time_gap = self._expand_batch(time_gap, batch_size, sample.device, sample.dtype) + if guidance_scale is None: + guidance_scale = torch.ones(batch_size, device=sample.device, dtype=sample.dtype) + else: + guidance_scale = self._expand_batch(guidance_scale, batch_size, sample.device, sample.dtype) + if guidance_interval_start is None: + guidance_interval_start = torch.zeros(batch_size, device=sample.device, dtype=sample.dtype) + else: + guidance_interval_start = self._expand_batch( + guidance_interval_start, batch_size, sample.device, sample.dtype + ) + if guidance_interval_end is None: + guidance_interval_end = torch.ones(batch_size, device=sample.device, dtype=sample.dtype) + else: + guidance_interval_end = self._expand_batch( + guidance_interval_end, batch_size, sample.device, sample.dtype + ) + + class_labels = class_labels.reshape(-1).long() + seq = self._build_sequence( + sample, + time_gap, + guidance_scale, + guidance_interval_start, + guidance_interval_end, + class_labels, + ) + + for block in self.shared_blocks: + if self.training and self.gradient_checkpointing: + seq = torch.utils.checkpoint.checkpoint(block, seq, self.rope_freqs, use_reentrant=False) + else: + seq = block(seq, self.rope_freqs) + + u_seq = v_seq = seq + for block in self.u_heads: + u_seq = block(u_seq, self.rope_freqs) + for block in self.v_heads: + v_seq = block(v_seq, self.rope_freqs) + + u_tokens = u_seq[:, self.prefix_tokens :] + velocity_u = self.unpatchify(self.u_final_layer(u_tokens)) + + velocity_v = None + if len(self.v_heads) > 0: + v_tokens = v_seq[:, self.prefix_tokens :] + velocity_v = self.unpatchify(self.v_final_layer(v_tokens)) + + if not return_dict: + if velocity_v is None: + return (velocity_u,) + return (velocity_u, velocity_v) + return IMFTransformer2DModelOutput(velocity_u=velocity_u, velocity_v=velocity_v) + + @staticmethod + def _expand_batch( + value: torch.Tensor, + batch_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + value = torch.as_tensor(value, device=device, dtype=dtype) + if value.ndim == 0: + value = value.reshape(1).repeat(batch_size) + else: + value = value.reshape(-1) + if value.shape[0] == 1 and batch_size > 1: + value = value.repeat(batch_size) + return value + + @classmethod + def from_imf_checkpoint( + cls, + checkpoint_path: str, + model_type: str | None = None, + map_location: str = "cpu", + strict: bool = True, + eval_mode: bool = True, + ) -> Tuple["IMFTransformer2DModel", Dict[str, object]]: + checkpoint = torch.load(checkpoint_path, map_location=map_location, weights_only=False) + if isinstance(checkpoint, Mapping) and "state_dict" in checkpoint: + state_dict = checkpoint["state_dict"] + elif isinstance(checkpoint, Mapping) and any(k.startswith("net.") for k in checkpoint): + state_dict = checkpoint + else: + state_dict = checkpoint + + inferred_type = model_type + if inferred_type is None and isinstance(checkpoint, Mapping): + args = checkpoint.get("args") + if isinstance(args, argparse.Namespace): + inferred_type = getattr(args, "model", None) or getattr(args, "model_str", None) + elif isinstance(args, Mapping): + inferred_type = args.get("model") or args.get("model_str") + + if inferred_type is None: + raise ValueError("model_type must be provided when loading a raw state dict checkpoint.") + + config = dict(IMF_PRESET_CONFIGS[inferred_type]) + config["model_type"] = inferred_type + config["eval_mode"] = eval_mode + model = cls(**config) + model.load_state_dict(remap_legacy_state_dict(state_dict), strict=strict) + metadata = {"checkpoint_path": checkpoint_path, "model_type": inferred_type} + return model, metadata + + def to_imf_checkpoint(self, prefix: str = "net.") -> Dict[str, torch.Tensor]: + state: Dict[str, torch.Tensor] = {} + for key, value in self.state_dict().items(): + if key == "rope_freqs": + continue + state[f"{prefix}{key}"] = value.detach().cpu() + return state + + @property + def net(self): + return self + + +IMFDiffusersModel = IMFTransformer2DModel diff --git a/iMF-XL-SIM/vae/config.json b/iMF-XL-SIM/vae/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ae2ab37de2acb8bc348d3de809ae8385324be962 --- /dev/null +++ b/iMF-XL-SIM/vae/config.json @@ -0,0 +1,38 @@ +{ + "_class_name": "AutoencoderKL", + "_diffusers_version": "0.36.0", + "_name_or_path": "stabilityai/sd-vae-ft-mse", + "act_fn": "silu", + "block_out_channels": [ + 128, + 256, + 512, + 512 + ], + "down_block_types": [ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D" + ], + "force_upcast": true, + "in_channels": 3, + "latent_channels": 4, + "latents_mean": null, + "latents_std": null, + "layers_per_block": 2, + "mid_block_add_attention": true, + "norm_num_groups": 32, + "out_channels": 3, + "sample_size": 256, + "scaling_factor": 0.18215, + "shift_factor": null, + "up_block_types": [ + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D" + ], + "use_post_quant_conv": true, + "use_quant_conv": true +} diff --git a/iMF-XL-SIM/vae/diffusion_pytorch_model.safetensors b/iMF-XL-SIM/vae/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..90464d67ac7303d0ee4696334df13da130a948ea --- /dev/null +++ b/iMF-XL-SIM/vae/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d993488569e928462932c8c38a0760b874d166399b14414135bd9c42df5815 +size 334643276 diff --git a/pMF-B-16-SIM/README.md b/pMF-B-16-SIM/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fca6d739c231d7326d4f68270bc8be567c58efa6 --- /dev/null +++ b/pMF-B-16-SIM/README.md @@ -0,0 +1,44 @@ +--- +license: mit +library_name: diffusers +pipeline_tag: text-to-image +tags: +- diffusers +- pmf +- image-generation +- class-conditional +- imagenet +inference: true +--- + +# pMF-B-16-SIM + +Self-contained Diffusers variant for **pMF-B/16 (FD-SIM post-trained)** (Pixel Mean Flows). + +Recommended settings: `guidance_scale=7.5`, interval `[0.1, 0.8]`, `noise_scale=1.0`. + +## Load + +```python +from pathlib import Path +from diffusers import DiffusionPipeline +import torch + +model_dir = Path("./pMF-B-16-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + local_files_only=True, + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.bfloat16, +).to("cuda") + +image = pipe( + class_labels=207, + num_inference_steps=1, + guidance_scale=7.5, + guidance_interval_min=0.1, + guidance_interval_max=0.8, + noise_scale=1.0, +).images[0] +``` diff --git a/pMF-B-16-SIM/model_index.json b/pMF-B-16-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..ec00b5bd38c72b4310e6e3cb2c65ce31ef2427b4 --- /dev/null +++ b/pMF-B-16-SIM/model_index.json @@ -0,0 +1,1017 @@ +{ + "_class_name": [ + "pipeline", + "PMFPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_pmf", + "PMFScheduler" + ], + "transformer": [ + "transformer_pmf", + "PMFTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + } +} diff --git a/pMF-B-16-SIM/pipeline.py b/pMF-B-16-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..314f675b581fe55ede8cbc0d1f2e1f850e5e9f34 --- /dev/null +++ b/pMF-B-16-SIM/pipeline.py @@ -0,0 +1,289 @@ +"""Hub custom pipeline: PMFPipeline.""" + +from __future__ import annotations + +import inspect +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +DEFAULT_CFG_BY_MODEL: Dict[str, Dict[str, float]] = { + "pMF-B/16": {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8}, + "pMF-B/32": {"guidance_scale": 6.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.7}, + "pMF-L/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.7}, + "pMF-L/32": {"guidance_scale": 7.5, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/32": {"guidance_scale": 5.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.6}, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + + +class PMFPipeline(DiffusionPipeline): + model_cpu_offload_seq = "transformer" + + def __init__( + self, + transformer: Any, + scheduler: Any, + id2label: Optional[Dict[Union[int, str], str]] = None, + ) -> None: + super().__init__() + if scheduler is None: + raise ValueError("PMFPipeline requires a scheduler loaded from the checkpoint.") + self.register_modules(transformer=transformer, scheduler=scheduler) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + + @staticmethod + def prepare_extra_step_kwargs( + scheduler: Any, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + eta: float | None = None, + ) -> Dict[str, Any]: + kwargs: Dict[str, Any] = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + @staticmethod + def _prepare_generator( + generator: Optional[Union[torch.Generator, List[torch.Generator]]], + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + """Reset generator state so repeated `__call__` with the same object is reproducible.""" + if generator is None: + return None + if isinstance(generator, list): + for gen in generator: + if gen is not None: + gen.manual_seed(int(gen.initial_seed())) + return generator + generator.manual_seed(int(generator.initial_seed())) + return generator + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + loaded = self._read_id2label_from_model_index(getattr(self.config, "_name_or_path", None)) + if loaded: + self._id2label = loaded + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _read_id2label_from_model_index(variant_path: Optional[str]) -> Dict[int, str]: + if not variant_path: + return {} + model_index_path = Path(variant_path).resolve() / "model_index.json" + if not model_index_path.exists(): + return {} + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if not isinstance(id2label, dict): + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + self._ensure_labels_loaded() + if not self.labels: + raise ValueError("No labels loaded. Ensure `id2label` exists in model_index.json.") + labels = [label] if isinstance(label, str) else label + missing = [item for item in labels if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Example valid labels: {preview}, ...") + return [self.labels[item] for item in labels] + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) # type: ignore[arg-type] + return [int(class_id) for class_id in class_labels] + + def _recommended_noise_scale(self) -> float: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in RECOMMENDED_NOISE_BY_MODEL: + return RECOMMENDED_NOISE_BY_MODEL[model_type] + image_size = int(self.transformer.config.sample_size) + return {256: 1.0, 512: 2.0}.get(image_size, 1.0) + + def _default_cfg(self) -> Dict[str, float]: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in DEFAULT_CFG_BY_MODEL: + return dict(DEFAULT_CFG_BY_MODEL[model_type]) + return {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8} + + def predict_u( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + guidance_interval_min: torch.Tensor, + guidance_interval_max: torch.Tensor, + ) -> torch.Tensor: + output = self.transformer( + sample=sample, + timestep=timestep, + class_labels=class_labels, + h=h, + omega=omega, + guidance_interval_min=guidance_interval_min, + guidance_interval_max=guidance_interval_max, + return_dict=True, + ) + return output.u + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + num_inference_steps: int = 1, + guidance_scale: Optional[float] = None, + guidance_interval_min: Optional[float] = None, + guidance_interval_max: Optional[float] = None, + noise_scale: Optional[float] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + if output_type not in {"pil", "np", "pt"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt'.") + + defaults = self._default_cfg() + if guidance_scale is None: + guidance_scale = defaults["guidance_scale"] + if guidance_interval_min is None: + guidance_interval_min = defaults["guidance_interval_min"] + if guidance_interval_max is None: + guidance_interval_max = defaults["guidance_interval_max"] + if noise_scale is None: + noise_scale = self._recommended_noise_scale() + + class_label_ids = self._normalize_class_labels(class_labels) + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + generator = self._prepare_generator(generator) + image_size = int(self.transformer.config.sample_size) + channels = int(self.transformer.config.in_channels) + null_class_val = int( + getattr(self.transformer.config, "num_classes", getattr(self.transformer.config, "num_class_embeds", 1000)) + ) + + latents = randn_tensor( + shape=(batch_size, channels, image_size, image_size), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) * noise_scale + + class_labels_t = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1) + class_labels_t = class_labels_t.clamp(0, null_class_val - 1) + + device = latents.device + dtype = latents.dtype + omega = torch.full((batch_size,), guidance_scale, device=device, dtype=dtype) + t_min = torch.full((batch_size,), guidance_interval_min, device=device, dtype=dtype) + t_max = torch.full((batch_size,), guidance_interval_max, device=device, dtype=dtype) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + timesteps = self.scheduler.timesteps + + for step_index in self.progress_bar(range(num_inference_steps)): + t = timesteps[step_index] + t_next = timesteps[step_index + 1] + h = (t - t_next).expand(batch_size).to(device=device, dtype=dtype) + t_batch = t.expand(batch_size).to(device=device, dtype=dtype) + + u = self.predict_u( + sample=latents, + timestep=t_batch, + class_labels=class_labels_t, + h=h, + omega=omega, + guidance_interval_min=t_min, + guidance_interval_max=t_max, + ) + latents = self.scheduler.step(u, t, latents, **extra_step_kwargs).prev_sample + + images_pt = ((latents.float().clamp(-1, 1) + 1.0) / 2.0).cpu() + if output_type == "pt": + images = images_pt + elif output_type == "np": + images = images_pt.permute(0, 2, 3, 1).numpy() + else: + images = self.numpy_to_pil(images_pt.permute(0, 2, 3, 1).numpy()) + + self.maybe_free_model_hooks() + + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) + + +PMFPipelineOutput = ImagePipelineOutput + +__all__ = ["PMFPipeline", "PMFPipelineOutput"] diff --git a/pMF-B-16-SIM/scheduler/scheduler_config.json b/pMF-B-16-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..341ff049a80407b660e0c785f274b39768b8ebce --- /dev/null +++ b/pMF-B-16-SIM/scheduler/scheduler_config.json @@ -0,0 +1,6 @@ +{ + "_class_name": "PMFScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000, + "stochastic_sampling": false +} diff --git a/pMF-B-16-SIM/scheduler/scheduling_pmf.py b/pMF-B-16-SIM/scheduler/scheduling_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..e4a599c65bb4c64bb5296ab66e5495ac0d936139 --- /dev/null +++ b/pMF-B-16-SIM/scheduler/scheduling_pmf.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class PMFSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class PMFScheduler(SchedulerMixin, ConfigMixin): + """Scheduler for Pixel Mean Flow sampling (t: 1 -> 0).""" + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + stochastic_sampling: bool = False, + ): + del num_train_timesteps + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> None: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.flatten()[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + @staticmethod + def _expand_time(value: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + value = value.to(dtype=sample.dtype, device=sample.device) + while value.ndim < sample.ndim: + value = value.unsqueeze(-1) + return value + + def _sample_noise( + self, + sample: torch.Tensor, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> torch.Tensor: + return randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[PMFSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = self._expand_time(t - t_next, sample) + + if self.config.stochastic_sampling: + t_expanded = self._expand_time(t, sample) + t_next_expanded = self._expand_time(t_next, sample) + x0 = sample - t_expanded * model_output + noise = self._sample_noise(sample, generator=generator) + prev_sample = (1.0 - t_next_expanded) * x0 + t_next_expanded * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return PMFSchedulerOutput(prev_sample=prev_sample) diff --git a/pMF-B-16-SIM/transformer/config.json b/pMF-B-16-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..399ff5dec228827c0727e9feeb796633ef96efde --- /dev/null +++ b/pMF-B-16-SIM/transformer/config.json @@ -0,0 +1,26 @@ +{ + "_class_name": "PMFTransformer2DModel", + "_diffusers_version": "0.38.0", + "aux_head_depth": 8, + "bottleneck_dim": 128, + "depth": 16, + "embedding_init_constant": 1.0, + "eval_mode": true, + "hidden_size": 768, + "in_channels": 3, + "mlp_ratio": 2.6666666666666665, + "model_type": "pMF-B/16", + "norm_eps": 1e-06, + "num_attention_heads": 12, + "num_cfg_tokens": 4, + "num_class_embeds": null, + "num_class_tokens": 8, + "num_classes": 1000, + "num_interval_tokens": 2, + "num_time_tokens": 4, + "patch_size": 16, + "sample_size": 256, + "t_clip_min": 0.05, + "token_init_constant": 1.0, + "weight_init_constant": 0.32 +} diff --git a/pMF-B-16-SIM/transformer/diffusion_pytorch_model.safetensors b/pMF-B-16-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e80acb8dd332d0b97822947b84094d6bf5a214f7 --- /dev/null +++ b/pMF-B-16-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0c76ac6631f096e10571f2c098f19306160bed9b9de24a3d0e32f1077b1a529 +size 472960528 diff --git a/pMF-B-16-SIM/transformer/transformer_pmf.py b/pMF-B-16-SIM/transformer/transformer_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f97067569ec0a4d038069edca73b9c1d003596 --- /dev/null +++ b/pMF-B-16-SIM/transformer/transformer_pmf.py @@ -0,0 +1,665 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from math import sqrt +from typing import Dict, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm +from diffusers.utils import BaseOutput + + +PMF_PRESET_CONFIGS: Dict[str, Dict[str, object]] = { + "pMF-B/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-B/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-H/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, + "pMF-H/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + +# Legacy torch repo keys (pmfDiT_*) +LEGACY_MODEL_ALIASES: Dict[str, str] = { + "pmfDiT_B_16": "pMF-B/16", + "pmfDiT_B_32": "pMF-B/32", + "pmfDiT_L_16": "pMF-L/16", + "pmfDiT_L_32": "pMF-L/32", + "pmfDiT_H_16": "pMF-H/16", + "pmfDiT_H_32": "pMF-H/32", +} + + +@dataclass +class PMFTransformer2DOutput(BaseOutput): + u: torch.Tensor + v: Optional[torch.Tensor] = None + + +def remap_legacy_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Map wrapper/backbone keys from legacy checkpoints to native PMFTransformer2DModel keys.""" + remapped: Dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + for prefix in ("transformer.", "net."): + if new_key.startswith(prefix): + new_key = new_key[len(prefix) :] + break + # Official PyTorch checkpoints use TorchLinear/TorchEmbedding wrappers. + new_key = new_key.replace("._flax_linear", "").replace("._flax_embedding", "") + if new_key == "rope_freqs": + continue + remapped[new_key] = value + return remapped + + +def config_from_legacy(config: Dict[str, object]) -> Dict[str, object]: + """Build native config kwargs from a legacy config.json dict.""" + model_type = config.get("model_type") or config.get("model_name") or config.get("model_str") + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type not in PMF_PRESET_CONFIGS: + raise ValueError(f"Unknown pMF preset '{model_type}'. Known: {list(PMF_PRESET_CONFIGS)}") + + preset = dict(PMF_PRESET_CONFIGS[model_type]) + preset["num_classes"] = int(config.get("num_class_embeds") or config.get("num_classes") or 1000) + preset["model_type"] = model_type + if config.get("sample_size") is not None: + preset["sample_size"] = int(config["sample_size"]) + if config.get("eval_mode") is not None: + preset["eval_mode"] = bool(config["eval_mode"]) + return preset + + +def _scaled_linear( + in_features: int, + out_features: int, + *, + bias: bool = True, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + bias_init: str = "zeros", +) -> nn.Linear: + layer = nn.Linear(in_features, out_features, bias=bias) + if weight_init == "scaled_variance": + std = init_constant / sqrt(in_features) + nn.init.normal_(layer.weight, std=std) + elif weight_init == "zeros": + nn.init.zeros_(layer.weight) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + + if bias: + if bias_init == "zeros": + nn.init.zeros_(layer.bias) + else: + raise ValueError(f"Invalid bias_init: {bias_init}") + return layer + + +class PMFTimestepEmbedder(nn.Module): + def __init__( + self, + hidden_size: int, + frequency_embedding_size: int = 256, + init_constant: float = 1.0, + ): + super().__init__() + init_kwargs = dict( + out_features=hidden_size, + bias=True, + weight_init="scaled_variance", + init_constant=init_constant, + bias_init="zeros", + ) + self.mlp = nn.Sequential( + _scaled_linear(frequency_embedding_size, **init_kwargs), + nn.SiLU(), + _scaled_linear(hidden_size, **init_kwargs), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + half = dim // 2 + compute_dtype = t.dtype + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=compute_dtype, device=t.device) + / half + ) + args = t[:, None] * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + return self.mlp(t_freq) + + +class PMFLabelEmbedder(nn.Module): + def __init__(self, num_classes: int, hidden_size: int, init_constant: float = 1.0): + super().__init__() + self.embedding_table = nn.Embedding(num_classes + 1, hidden_size) + nn.init.normal_(self.embedding_table.weight, std=init_constant / sqrt(hidden_size)) + + def forward(self, labels: torch.Tensor) -> torch.Tensor: + return self.embedding_table(labels) + + +class PMFBottleneckPatchEmbedder(nn.Module): + def __init__( + self, + input_size: int, + patch_size: int, + pca_channels: int, + in_channels: int, + hidden_size: int, + bias: bool = True, + ): + super().__init__() + self.patch_size = (patch_size, patch_size) + self.num_patches = (input_size // patch_size) ** 2 + self.proj1 = nn.Conv2d( + in_channels, + pca_channels, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + ) + self.proj2 = nn.Conv2d(pca_channels, hidden_size, kernel_size=1, stride=1, bias=bias) + + kh = kw = patch_size + fan_in = kh * kw * in_channels + fan_out = pca_channels + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj1.weight, -limit, limit) + fan_in = pca_channels + fan_out = hidden_size + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj2.weight, -limit, limit) + if bias: + nn.init.zeros_(self.proj1.bias) + nn.init.zeros_(self.proj2.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj2(self.proj1(x)) + return x.flatten(2).transpose(1, 2) + + +def precompute_rope_freqs(dim: int, seq_len: int, theta: float = 10000.0) -> torch.Tensor: + dim = dim // 2 + grid_size = int(seq_len**0.5) + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + positions = torch.arange(grid_size, dtype=torch.float32) + freqs_h = torch.einsum("i,j->ij", positions, freqs) + freqs_w = torch.einsum("i,j->ij", positions, freqs) + freqs_2d = torch.cat( + [ + torch.tile(freqs_h[:, None, :], (1, grid_size, 1)), + torch.tile(freqs_w[None, :, :], (grid_size, 1, 1)), + ], + dim=-1, + ) + real = torch.cos(freqs_2d).reshape(seq_len, dim) + imag = torch.sin(freqs_2d).reshape(seq_len, dim) + return torch.complex(real, imag) + + +def apply_rotary_pos_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + x_float = x.to(torch.float32) + x_complex = torch.view_as_complex(x_float.reshape(*x_float.shape[:-1], -1, 2).contiguous()) + freqs_cis = freqs_cis.unsqueeze(0).unsqueeze(2) + token_count = freqs_cis.shape[1] + x_rotated = x_complex.clone() + x_rotated[:, -token_count:, :] = x_complex[:, -token_count:, :] * freqs_cis + x_out = torch.view_as_real(x_rotated).flatten(-2) + return x_out.to(x.dtype) + + +class PMFAttention(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + init_kwargs = dict( + bias=False, + weight_init="scaled_variance", + init_constant=weight_init_constant, + ) + self.q_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.k_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.v_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.out_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.q_norm = RMSNorm(self.head_dim, eps=eps) + self.k_norm = RMSNorm(self.head_dim, eps=eps) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + batch_size, seq_len, channels = x.shape + q = self.q_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + k = self.k_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + v = self.v_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + + q = self.q_norm(q) + k = self.k_norm(k) + q = apply_rotary_pos_emb(q, rope_freqs) + k = apply_rotary_pos_emb(k, rope_freqs) + + query = q / math.sqrt(self.head_dim) + attn_weights = torch.einsum("bqhd,bkhd->bhqk", query, k) + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(v.dtype) + attn = torch.einsum("bhqk,bkhd->bqhd", attn_weights, v) + attn = attn.reshape(batch_size, seq_len, channels) + return self.out_proj(attn) + + +class PMFSwiGLUMlp(nn.Module): + def __init__(self, dim: int, hidden_dim: int, weight_init_constant: float = 0.32): + super().__init__() + init_kwargs = dict(bias=False, weight_init="scaled_variance", init_constant=weight_init_constant) + self.w1 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w3 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w2 = _scaled_linear(hidden_dim, dim, **init_kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + + +class PMFTransformerBlock(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float = 8 / 3, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.norm1 = RMSNorm(hidden_size, eps=eps) + self.attn = PMFAttention(hidden_size, num_heads, weight_init_constant=weight_init_constant, eps=eps) + self.norm2 = RMSNorm(hidden_size, eps=eps) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + if hidden_size > 1024: + mlp_hidden_dim = (mlp_hidden_dim + 7) // 8 * 8 + self.mlp = PMFSwiGLUMlp(hidden_size, mlp_hidden_dim, weight_init_constant=weight_init_constant) + self.attn_scale = nn.Parameter(torch.zeros(hidden_size)) + self.mlp_scale = nn.Parameter(torch.zeros(hidden_size)) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), rope_freqs) * self.attn_scale + x = x + self.mlp(self.norm2(x)) * self.mlp_scale + return x + + +class PMFFinalLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int, eps: float = 1e-6): + super().__init__() + self.norm = RMSNorm(hidden_size, eps=eps) + self.linear = _scaled_linear( + hidden_size, + patch_size * patch_size * out_channels, + bias=True, + weight_init="zeros", + bias_init="zeros", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(self.norm(x)) + + +class PMFTransformer2DModel(ModelMixin, ConfigMixin): + """Native diffusers implementation of the pMF DiT backbone.""" + + _supports_gradient_checkpointing = True + _skip_layerwise_casting_patterns = ["pos_embed", "rope_freqs"] + + @register_to_config + def __init__( + self, + sample_size: int = 256, + patch_size: int = 16, + in_channels: int = 3, + hidden_size: int = 768, + depth: int = 16, + num_attention_heads: int = 12, + mlp_ratio: float = 8 / 3, + num_classes: int = 1000, + bottleneck_dim: int = 128, + aux_head_depth: int = 8, + num_class_tokens: int = 8, + num_time_tokens: int = 4, + num_cfg_tokens: int = 4, + num_interval_tokens: int = 2, + token_init_constant: float = 1.0, + embedding_init_constant: float = 1.0, + weight_init_constant: float = 0.32, + eval_mode: bool = True, + model_type: str | None = None, + num_class_embeds: int | None = None, + t_clip_min: float = 0.05, + norm_eps: float = 1e-6, + ): + super().__init__() + if num_class_embeds is not None: + num_classes = int(num_class_embeds) + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type in PMF_PRESET_CONFIGS: + preset = PMF_PRESET_CONFIGS[model_type] + sample_size = int(preset["sample_size"]) + patch_size = int(preset["patch_size"]) + hidden_size = int(preset["hidden_size"]) + depth = int(preset["depth"]) + num_attention_heads = int(preset["num_attention_heads"]) + bottleneck_dim = int(preset["bottleneck_dim"]) + aux_head_depth = int(preset["aux_head_depth"]) + + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.depth = depth + self.num_attention_heads = num_attention_heads + self.aux_head_depth = aux_head_depth + self.num_class_tokens = num_class_tokens + self.num_time_tokens = num_time_tokens + self.num_cfg_tokens = num_cfg_tokens + self.num_interval_tokens = num_interval_tokens + self.prefix_tokens = ( + num_class_tokens + num_cfg_tokens + 2 * num_interval_tokens + num_time_tokens + ) + self.t_clip_min = t_clip_min + self.eval_mode = eval_mode + self.gradient_checkpointing = False + + self.x_embedder = PMFBottleneckPatchEmbedder( + sample_size, + patch_size, + bottleneck_dim, + in_channels, + hidden_size, + bias=True, + ) + embed_kwargs = dict(hidden_size=hidden_size, init_constant=embedding_init_constant) + self.h_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.omega_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_start_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_end_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.y_embedder = PMFLabelEmbedder(num_classes, hidden_size, init_constant=embedding_init_constant) + + token_std = token_init_constant / math.sqrt(hidden_size) + self.time_tokens = nn.Parameter(torch.randn(1, num_time_tokens, hidden_size) * token_std) + self.class_tokens = nn.Parameter(torch.randn(1, num_class_tokens, hidden_size) * token_std) + self.omega_tokens = nn.Parameter(torch.randn(1, num_cfg_tokens, hidden_size) * token_std) + self.t_min_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + self.t_max_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + + total_tokens = self.x_embedder.num_patches + self.prefix_tokens + self.pos_embed = nn.Parameter(torch.randn(1, total_tokens, hidden_size) * 0.02) + + head_dim = hidden_size // num_attention_heads + self.register_buffer( + "rope_freqs", + precompute_rope_freqs(head_dim, self.x_embedder.num_patches), + persistent=False, + ) + + shared_depth = depth - aux_head_depth + block_kwargs = dict( + hidden_size=hidden_size, + num_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + weight_init_constant=weight_init_constant, + eps=norm_eps, + ) + self.shared_blocks = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(shared_depth)]) + self.u_heads = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth)]) + self.v_heads = nn.ModuleList( + [PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth if not eval_mode else 0)] + ) + self.u_final_layer = PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + self.v_final_layer = ( + PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + if not eval_mode + else None + ) + + def _build_sequence( + self, + sample: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + t_min: torch.Tensor, + t_max: torch.Tensor, + class_labels: torch.Tensor, + ) -> torch.Tensor: + x_embed = self.x_embedder(sample) + h_embed = self.h_embedder(h) + omega_embed = self.omega_embedder(1 - 1 / omega) + t_min_embed = self.cfg_t_start_embedder(t_min) + t_max_embed = self.cfg_t_end_embedder(t_max) + y_embed = self.y_embedder(class_labels) + + time_tokens = self.time_tokens + h_embed.unsqueeze(1) + omega_tokens = self.omega_tokens + omega_embed.unsqueeze(1) + t_min_tokens = self.t_min_tokens + t_min_embed.unsqueeze(1) + t_max_tokens = self.t_max_tokens + t_max_embed.unsqueeze(1) + class_tokens = self.class_tokens + y_embed.unsqueeze(1) + + seq = torch.cat( + [class_tokens, omega_tokens, t_min_tokens, t_max_tokens, time_tokens, x_embed], + dim=1, + ) + return seq + self.pos_embed + + def _unpatchify(self, tokens: torch.Tensor) -> torch.Tensor: + batch_size = tokens.shape[0] + patch = self.patch_size + grid = int(tokens.shape[1] ** 0.5) + channels = self.out_channels + x = tokens.reshape(batch_size, grid, grid, patch, patch, channels) + x = torch.einsum("nhwpqc->nchpwq", x) + return x.reshape(batch_size, channels, grid * patch, grid * patch) + + def forward( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: Optional[torch.Tensor] = None, + omega: Optional[torch.Tensor] = None, + guidance_interval_min: Optional[torch.Tensor] = None, + guidance_interval_max: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> PMFTransformer2DOutput | Tuple[torch.Tensor, Optional[torch.Tensor]]: + batch_size = sample.shape[0] + timestep = self._expand_batch(timestep, batch_size, sample.device, sample.dtype) + h = self._expand_batch(h if h is not None else timestep, batch_size, sample.device, sample.dtype) + omega = self._expand_batch( + omega if omega is not None else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_min = self._expand_batch( + guidance_interval_min + if guidance_interval_min is not None + else torch.zeros(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_max = self._expand_batch( + guidance_interval_max + if guidance_interval_max is not None + else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + + seq = self._build_sequence(sample, h, omega, guidance_interval_min, guidance_interval_max, class_labels) + rope_freqs = self.rope_freqs.to(device=sample.device) + + for block in self.shared_blocks: + if self.training and self.gradient_checkpointing: + seq = torch.utils.checkpoint.checkpoint(block, seq, rope_freqs, use_reentrant=False) + else: + seq = block(seq, rope_freqs) + + u_seq = v_seq = seq + for block in self.u_heads: + if self.training and self.gradient_checkpointing: + u_seq = torch.utils.checkpoint.checkpoint(block, u_seq, rope_freqs, use_reentrant=False) + else: + u_seq = block(u_seq, rope_freqs) + + for block in self.v_heads: + if self.training and self.gradient_checkpointing: + v_seq = torch.utils.checkpoint.checkpoint(block, v_seq, rope_freqs, use_reentrant=False) + else: + v_seq = block(v_seq, rope_freqs) + + u_tokens = u_seq[:, self.prefix_tokens :] + u_pred = self._unpatchify(self.u_final_layer(u_tokens)) + t = timestep.reshape(batch_size, 1, 1, 1) + u = (sample - u_pred) / torch.clamp(t, min=self.t_clip_min) + + v = None + if self.v_final_layer is not None: + v_tokens = v_seq[:, self.prefix_tokens :] + v_pred = self._unpatchify(self.v_final_layer(v_tokens)) + v = (sample - v_pred) / torch.clamp(t, min=self.t_clip_min) + + if not return_dict: + return (u, v) + return PMFTransformer2DOutput(u=u, v=v) + + @staticmethod + def _expand_batch( + value: torch.Tensor, + batch_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + value = torch.as_tensor(value, device=device, dtype=dtype) + if value.ndim == 0: + value = value.reshape(1) + if value.shape[0] == 1 and batch_size > 1: + value = value.expand(batch_size) + return value.reshape(batch_size) + + @classmethod + def from_pmf_checkpoint( + cls, + checkpoint_path: str, + model_type: str | None = None, + map_location: str = "cpu", + strict: bool = False, + ) -> Tuple["PMFTransformer2DModel", Dict[str, object]]: + checkpoint = torch.load(checkpoint_path, map_location=map_location, weights_only=False) + if isinstance(checkpoint, dict) and "state_dict" in checkpoint: + state_dict = checkpoint["state_dict"] + else: + state_dict = checkpoint + + if model_type is None: + for key in ("model_type", "model_str", "model"): + if isinstance(checkpoint, dict) and key in checkpoint: + model_type = checkpoint[key] + break + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type is None: + raise ValueError("model_type is required when it cannot be inferred from the checkpoint.") + + config = dict(PMF_PRESET_CONFIGS[model_type]) + config["model_type"] = model_type + config["eval_mode"] = True + model = cls(**config) + model.load_state_dict(remap_legacy_state_dict(state_dict), strict=strict) + metadata = {"checkpoint_path": checkpoint_path, "model_type": model_type} + return model, metadata + + def to_pmf_checkpoint(self, prefix: str = "net.") -> Dict[str, torch.Tensor]: + state_dict: Dict[str, torch.Tensor] = {} + for key, value in self.state_dict().items(): + state_dict[f"{prefix}{key}"] = value.detach().cpu() + return state_dict + + @property + def net(self): + return self + + +PMFDiffusersModel = PMFTransformer2DModel diff --git a/pMF-B-32-SIM/README.md b/pMF-B-32-SIM/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2071141a37b38c13124fda6f4e4fb9646ab3b958 --- /dev/null +++ b/pMF-B-32-SIM/README.md @@ -0,0 +1,44 @@ +--- +license: mit +library_name: diffusers +pipeline_tag: text-to-image +tags: +- diffusers +- pmf +- image-generation +- class-conditional +- imagenet +inference: true +--- + +# pMF-B-32-SIM + +Self-contained Diffusers variant for **pMF-B/32 (FD-SIM post-trained)** (Pixel Mean Flows). + +Recommended settings: `guidance_scale=6.5`, interval `[0.1, 0.7]`, `noise_scale=2.0`. + +## Load + +```python +from pathlib import Path +from diffusers import DiffusionPipeline +import torch + +model_dir = Path("./pMF-B-32-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + local_files_only=True, + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.bfloat16, +).to("cuda") + +image = pipe( + class_labels=207, + num_inference_steps=1, + guidance_scale=6.5, + guidance_interval_min=0.1, + guidance_interval_max=0.7, + noise_scale=2.0, +).images[0] +``` diff --git a/pMF-B-32-SIM/model_index.json b/pMF-B-32-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..ec00b5bd38c72b4310e6e3cb2c65ce31ef2427b4 --- /dev/null +++ b/pMF-B-32-SIM/model_index.json @@ -0,0 +1,1017 @@ +{ + "_class_name": [ + "pipeline", + "PMFPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_pmf", + "PMFScheduler" + ], + "transformer": [ + "transformer_pmf", + "PMFTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + } +} diff --git a/pMF-B-32-SIM/pipeline.py b/pMF-B-32-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..314f675b581fe55ede8cbc0d1f2e1f850e5e9f34 --- /dev/null +++ b/pMF-B-32-SIM/pipeline.py @@ -0,0 +1,289 @@ +"""Hub custom pipeline: PMFPipeline.""" + +from __future__ import annotations + +import inspect +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +DEFAULT_CFG_BY_MODEL: Dict[str, Dict[str, float]] = { + "pMF-B/16": {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8}, + "pMF-B/32": {"guidance_scale": 6.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.7}, + "pMF-L/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.7}, + "pMF-L/32": {"guidance_scale": 7.5, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/32": {"guidance_scale": 5.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.6}, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + + +class PMFPipeline(DiffusionPipeline): + model_cpu_offload_seq = "transformer" + + def __init__( + self, + transformer: Any, + scheduler: Any, + id2label: Optional[Dict[Union[int, str], str]] = None, + ) -> None: + super().__init__() + if scheduler is None: + raise ValueError("PMFPipeline requires a scheduler loaded from the checkpoint.") + self.register_modules(transformer=transformer, scheduler=scheduler) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + + @staticmethod + def prepare_extra_step_kwargs( + scheduler: Any, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + eta: float | None = None, + ) -> Dict[str, Any]: + kwargs: Dict[str, Any] = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + @staticmethod + def _prepare_generator( + generator: Optional[Union[torch.Generator, List[torch.Generator]]], + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + """Reset generator state so repeated `__call__` with the same object is reproducible.""" + if generator is None: + return None + if isinstance(generator, list): + for gen in generator: + if gen is not None: + gen.manual_seed(int(gen.initial_seed())) + return generator + generator.manual_seed(int(generator.initial_seed())) + return generator + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + loaded = self._read_id2label_from_model_index(getattr(self.config, "_name_or_path", None)) + if loaded: + self._id2label = loaded + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _read_id2label_from_model_index(variant_path: Optional[str]) -> Dict[int, str]: + if not variant_path: + return {} + model_index_path = Path(variant_path).resolve() / "model_index.json" + if not model_index_path.exists(): + return {} + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if not isinstance(id2label, dict): + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + self._ensure_labels_loaded() + if not self.labels: + raise ValueError("No labels loaded. Ensure `id2label` exists in model_index.json.") + labels = [label] if isinstance(label, str) else label + missing = [item for item in labels if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Example valid labels: {preview}, ...") + return [self.labels[item] for item in labels] + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) # type: ignore[arg-type] + return [int(class_id) for class_id in class_labels] + + def _recommended_noise_scale(self) -> float: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in RECOMMENDED_NOISE_BY_MODEL: + return RECOMMENDED_NOISE_BY_MODEL[model_type] + image_size = int(self.transformer.config.sample_size) + return {256: 1.0, 512: 2.0}.get(image_size, 1.0) + + def _default_cfg(self) -> Dict[str, float]: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in DEFAULT_CFG_BY_MODEL: + return dict(DEFAULT_CFG_BY_MODEL[model_type]) + return {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8} + + def predict_u( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + guidance_interval_min: torch.Tensor, + guidance_interval_max: torch.Tensor, + ) -> torch.Tensor: + output = self.transformer( + sample=sample, + timestep=timestep, + class_labels=class_labels, + h=h, + omega=omega, + guidance_interval_min=guidance_interval_min, + guidance_interval_max=guidance_interval_max, + return_dict=True, + ) + return output.u + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + num_inference_steps: int = 1, + guidance_scale: Optional[float] = None, + guidance_interval_min: Optional[float] = None, + guidance_interval_max: Optional[float] = None, + noise_scale: Optional[float] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + if output_type not in {"pil", "np", "pt"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt'.") + + defaults = self._default_cfg() + if guidance_scale is None: + guidance_scale = defaults["guidance_scale"] + if guidance_interval_min is None: + guidance_interval_min = defaults["guidance_interval_min"] + if guidance_interval_max is None: + guidance_interval_max = defaults["guidance_interval_max"] + if noise_scale is None: + noise_scale = self._recommended_noise_scale() + + class_label_ids = self._normalize_class_labels(class_labels) + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + generator = self._prepare_generator(generator) + image_size = int(self.transformer.config.sample_size) + channels = int(self.transformer.config.in_channels) + null_class_val = int( + getattr(self.transformer.config, "num_classes", getattr(self.transformer.config, "num_class_embeds", 1000)) + ) + + latents = randn_tensor( + shape=(batch_size, channels, image_size, image_size), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) * noise_scale + + class_labels_t = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1) + class_labels_t = class_labels_t.clamp(0, null_class_val - 1) + + device = latents.device + dtype = latents.dtype + omega = torch.full((batch_size,), guidance_scale, device=device, dtype=dtype) + t_min = torch.full((batch_size,), guidance_interval_min, device=device, dtype=dtype) + t_max = torch.full((batch_size,), guidance_interval_max, device=device, dtype=dtype) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + timesteps = self.scheduler.timesteps + + for step_index in self.progress_bar(range(num_inference_steps)): + t = timesteps[step_index] + t_next = timesteps[step_index + 1] + h = (t - t_next).expand(batch_size).to(device=device, dtype=dtype) + t_batch = t.expand(batch_size).to(device=device, dtype=dtype) + + u = self.predict_u( + sample=latents, + timestep=t_batch, + class_labels=class_labels_t, + h=h, + omega=omega, + guidance_interval_min=t_min, + guidance_interval_max=t_max, + ) + latents = self.scheduler.step(u, t, latents, **extra_step_kwargs).prev_sample + + images_pt = ((latents.float().clamp(-1, 1) + 1.0) / 2.0).cpu() + if output_type == "pt": + images = images_pt + elif output_type == "np": + images = images_pt.permute(0, 2, 3, 1).numpy() + else: + images = self.numpy_to_pil(images_pt.permute(0, 2, 3, 1).numpy()) + + self.maybe_free_model_hooks() + + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) + + +PMFPipelineOutput = ImagePipelineOutput + +__all__ = ["PMFPipeline", "PMFPipelineOutput"] diff --git a/pMF-B-32-SIM/scheduler/scheduler_config.json b/pMF-B-32-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..341ff049a80407b660e0c785f274b39768b8ebce --- /dev/null +++ b/pMF-B-32-SIM/scheduler/scheduler_config.json @@ -0,0 +1,6 @@ +{ + "_class_name": "PMFScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000, + "stochastic_sampling": false +} diff --git a/pMF-B-32-SIM/scheduler/scheduling_pmf.py b/pMF-B-32-SIM/scheduler/scheduling_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..e4a599c65bb4c64bb5296ab66e5495ac0d936139 --- /dev/null +++ b/pMF-B-32-SIM/scheduler/scheduling_pmf.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class PMFSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class PMFScheduler(SchedulerMixin, ConfigMixin): + """Scheduler for Pixel Mean Flow sampling (t: 1 -> 0).""" + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + stochastic_sampling: bool = False, + ): + del num_train_timesteps + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> None: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.flatten()[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + @staticmethod + def _expand_time(value: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + value = value.to(dtype=sample.dtype, device=sample.device) + while value.ndim < sample.ndim: + value = value.unsqueeze(-1) + return value + + def _sample_noise( + self, + sample: torch.Tensor, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> torch.Tensor: + return randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[PMFSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = self._expand_time(t - t_next, sample) + + if self.config.stochastic_sampling: + t_expanded = self._expand_time(t, sample) + t_next_expanded = self._expand_time(t_next, sample) + x0 = sample - t_expanded * model_output + noise = self._sample_noise(sample, generator=generator) + prev_sample = (1.0 - t_next_expanded) * x0 + t_next_expanded * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return PMFSchedulerOutput(prev_sample=prev_sample) diff --git a/pMF-B-32-SIM/transformer/config.json b/pMF-B-32-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c4c23ccf3eaf47867b3dbc25030c04c9c3745772 --- /dev/null +++ b/pMF-B-32-SIM/transformer/config.json @@ -0,0 +1,26 @@ +{ + "_class_name": "PMFTransformer2DModel", + "_diffusers_version": "0.38.0", + "aux_head_depth": 8, + "bottleneck_dim": 128, + "depth": 16, + "embedding_init_constant": 1.0, + "eval_mode": true, + "hidden_size": 768, + "in_channels": 3, + "mlp_ratio": 2.6666666666666665, + "model_type": "pMF-B/32", + "norm_eps": 1e-06, + "num_attention_heads": 12, + "num_cfg_tokens": 4, + "num_class_embeds": null, + "num_class_tokens": 8, + "num_classes": 1000, + "num_interval_tokens": 2, + "num_time_tokens": 4, + "patch_size": 32, + "sample_size": 512, + "t_clip_min": 0.05, + "token_init_constant": 1.0, + "weight_init_constant": 0.32 +} diff --git a/pMF-B-32-SIM/transformer/diffusion_pytorch_model.safetensors b/pMF-B-32-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..161080d30719987923ba08900facb13709e02433 --- /dev/null +++ b/pMF-B-32-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d42e422f6827d7e9e0ff48953e10693c5ecac1e77df335878b01918d62b8ae6 +size 481227280 diff --git a/pMF-B-32-SIM/transformer/transformer_pmf.py b/pMF-B-32-SIM/transformer/transformer_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f97067569ec0a4d038069edca73b9c1d003596 --- /dev/null +++ b/pMF-B-32-SIM/transformer/transformer_pmf.py @@ -0,0 +1,665 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from math import sqrt +from typing import Dict, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm +from diffusers.utils import BaseOutput + + +PMF_PRESET_CONFIGS: Dict[str, Dict[str, object]] = { + "pMF-B/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-B/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-H/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, + "pMF-H/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + +# Legacy torch repo keys (pmfDiT_*) +LEGACY_MODEL_ALIASES: Dict[str, str] = { + "pmfDiT_B_16": "pMF-B/16", + "pmfDiT_B_32": "pMF-B/32", + "pmfDiT_L_16": "pMF-L/16", + "pmfDiT_L_32": "pMF-L/32", + "pmfDiT_H_16": "pMF-H/16", + "pmfDiT_H_32": "pMF-H/32", +} + + +@dataclass +class PMFTransformer2DOutput(BaseOutput): + u: torch.Tensor + v: Optional[torch.Tensor] = None + + +def remap_legacy_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Map wrapper/backbone keys from legacy checkpoints to native PMFTransformer2DModel keys.""" + remapped: Dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + for prefix in ("transformer.", "net."): + if new_key.startswith(prefix): + new_key = new_key[len(prefix) :] + break + # Official PyTorch checkpoints use TorchLinear/TorchEmbedding wrappers. + new_key = new_key.replace("._flax_linear", "").replace("._flax_embedding", "") + if new_key == "rope_freqs": + continue + remapped[new_key] = value + return remapped + + +def config_from_legacy(config: Dict[str, object]) -> Dict[str, object]: + """Build native config kwargs from a legacy config.json dict.""" + model_type = config.get("model_type") or config.get("model_name") or config.get("model_str") + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type not in PMF_PRESET_CONFIGS: + raise ValueError(f"Unknown pMF preset '{model_type}'. Known: {list(PMF_PRESET_CONFIGS)}") + + preset = dict(PMF_PRESET_CONFIGS[model_type]) + preset["num_classes"] = int(config.get("num_class_embeds") or config.get("num_classes") or 1000) + preset["model_type"] = model_type + if config.get("sample_size") is not None: + preset["sample_size"] = int(config["sample_size"]) + if config.get("eval_mode") is not None: + preset["eval_mode"] = bool(config["eval_mode"]) + return preset + + +def _scaled_linear( + in_features: int, + out_features: int, + *, + bias: bool = True, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + bias_init: str = "zeros", +) -> nn.Linear: + layer = nn.Linear(in_features, out_features, bias=bias) + if weight_init == "scaled_variance": + std = init_constant / sqrt(in_features) + nn.init.normal_(layer.weight, std=std) + elif weight_init == "zeros": + nn.init.zeros_(layer.weight) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + + if bias: + if bias_init == "zeros": + nn.init.zeros_(layer.bias) + else: + raise ValueError(f"Invalid bias_init: {bias_init}") + return layer + + +class PMFTimestepEmbedder(nn.Module): + def __init__( + self, + hidden_size: int, + frequency_embedding_size: int = 256, + init_constant: float = 1.0, + ): + super().__init__() + init_kwargs = dict( + out_features=hidden_size, + bias=True, + weight_init="scaled_variance", + init_constant=init_constant, + bias_init="zeros", + ) + self.mlp = nn.Sequential( + _scaled_linear(frequency_embedding_size, **init_kwargs), + nn.SiLU(), + _scaled_linear(hidden_size, **init_kwargs), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + half = dim // 2 + compute_dtype = t.dtype + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=compute_dtype, device=t.device) + / half + ) + args = t[:, None] * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + return self.mlp(t_freq) + + +class PMFLabelEmbedder(nn.Module): + def __init__(self, num_classes: int, hidden_size: int, init_constant: float = 1.0): + super().__init__() + self.embedding_table = nn.Embedding(num_classes + 1, hidden_size) + nn.init.normal_(self.embedding_table.weight, std=init_constant / sqrt(hidden_size)) + + def forward(self, labels: torch.Tensor) -> torch.Tensor: + return self.embedding_table(labels) + + +class PMFBottleneckPatchEmbedder(nn.Module): + def __init__( + self, + input_size: int, + patch_size: int, + pca_channels: int, + in_channels: int, + hidden_size: int, + bias: bool = True, + ): + super().__init__() + self.patch_size = (patch_size, patch_size) + self.num_patches = (input_size // patch_size) ** 2 + self.proj1 = nn.Conv2d( + in_channels, + pca_channels, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + ) + self.proj2 = nn.Conv2d(pca_channels, hidden_size, kernel_size=1, stride=1, bias=bias) + + kh = kw = patch_size + fan_in = kh * kw * in_channels + fan_out = pca_channels + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj1.weight, -limit, limit) + fan_in = pca_channels + fan_out = hidden_size + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj2.weight, -limit, limit) + if bias: + nn.init.zeros_(self.proj1.bias) + nn.init.zeros_(self.proj2.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj2(self.proj1(x)) + return x.flatten(2).transpose(1, 2) + + +def precompute_rope_freqs(dim: int, seq_len: int, theta: float = 10000.0) -> torch.Tensor: + dim = dim // 2 + grid_size = int(seq_len**0.5) + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + positions = torch.arange(grid_size, dtype=torch.float32) + freqs_h = torch.einsum("i,j->ij", positions, freqs) + freqs_w = torch.einsum("i,j->ij", positions, freqs) + freqs_2d = torch.cat( + [ + torch.tile(freqs_h[:, None, :], (1, grid_size, 1)), + torch.tile(freqs_w[None, :, :], (grid_size, 1, 1)), + ], + dim=-1, + ) + real = torch.cos(freqs_2d).reshape(seq_len, dim) + imag = torch.sin(freqs_2d).reshape(seq_len, dim) + return torch.complex(real, imag) + + +def apply_rotary_pos_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + x_float = x.to(torch.float32) + x_complex = torch.view_as_complex(x_float.reshape(*x_float.shape[:-1], -1, 2).contiguous()) + freqs_cis = freqs_cis.unsqueeze(0).unsqueeze(2) + token_count = freqs_cis.shape[1] + x_rotated = x_complex.clone() + x_rotated[:, -token_count:, :] = x_complex[:, -token_count:, :] * freqs_cis + x_out = torch.view_as_real(x_rotated).flatten(-2) + return x_out.to(x.dtype) + + +class PMFAttention(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + init_kwargs = dict( + bias=False, + weight_init="scaled_variance", + init_constant=weight_init_constant, + ) + self.q_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.k_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.v_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.out_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.q_norm = RMSNorm(self.head_dim, eps=eps) + self.k_norm = RMSNorm(self.head_dim, eps=eps) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + batch_size, seq_len, channels = x.shape + q = self.q_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + k = self.k_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + v = self.v_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + + q = self.q_norm(q) + k = self.k_norm(k) + q = apply_rotary_pos_emb(q, rope_freqs) + k = apply_rotary_pos_emb(k, rope_freqs) + + query = q / math.sqrt(self.head_dim) + attn_weights = torch.einsum("bqhd,bkhd->bhqk", query, k) + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(v.dtype) + attn = torch.einsum("bhqk,bkhd->bqhd", attn_weights, v) + attn = attn.reshape(batch_size, seq_len, channels) + return self.out_proj(attn) + + +class PMFSwiGLUMlp(nn.Module): + def __init__(self, dim: int, hidden_dim: int, weight_init_constant: float = 0.32): + super().__init__() + init_kwargs = dict(bias=False, weight_init="scaled_variance", init_constant=weight_init_constant) + self.w1 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w3 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w2 = _scaled_linear(hidden_dim, dim, **init_kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + + +class PMFTransformerBlock(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float = 8 / 3, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.norm1 = RMSNorm(hidden_size, eps=eps) + self.attn = PMFAttention(hidden_size, num_heads, weight_init_constant=weight_init_constant, eps=eps) + self.norm2 = RMSNorm(hidden_size, eps=eps) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + if hidden_size > 1024: + mlp_hidden_dim = (mlp_hidden_dim + 7) // 8 * 8 + self.mlp = PMFSwiGLUMlp(hidden_size, mlp_hidden_dim, weight_init_constant=weight_init_constant) + self.attn_scale = nn.Parameter(torch.zeros(hidden_size)) + self.mlp_scale = nn.Parameter(torch.zeros(hidden_size)) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), rope_freqs) * self.attn_scale + x = x + self.mlp(self.norm2(x)) * self.mlp_scale + return x + + +class PMFFinalLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int, eps: float = 1e-6): + super().__init__() + self.norm = RMSNorm(hidden_size, eps=eps) + self.linear = _scaled_linear( + hidden_size, + patch_size * patch_size * out_channels, + bias=True, + weight_init="zeros", + bias_init="zeros", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(self.norm(x)) + + +class PMFTransformer2DModel(ModelMixin, ConfigMixin): + """Native diffusers implementation of the pMF DiT backbone.""" + + _supports_gradient_checkpointing = True + _skip_layerwise_casting_patterns = ["pos_embed", "rope_freqs"] + + @register_to_config + def __init__( + self, + sample_size: int = 256, + patch_size: int = 16, + in_channels: int = 3, + hidden_size: int = 768, + depth: int = 16, + num_attention_heads: int = 12, + mlp_ratio: float = 8 / 3, + num_classes: int = 1000, + bottleneck_dim: int = 128, + aux_head_depth: int = 8, + num_class_tokens: int = 8, + num_time_tokens: int = 4, + num_cfg_tokens: int = 4, + num_interval_tokens: int = 2, + token_init_constant: float = 1.0, + embedding_init_constant: float = 1.0, + weight_init_constant: float = 0.32, + eval_mode: bool = True, + model_type: str | None = None, + num_class_embeds: int | None = None, + t_clip_min: float = 0.05, + norm_eps: float = 1e-6, + ): + super().__init__() + if num_class_embeds is not None: + num_classes = int(num_class_embeds) + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type in PMF_PRESET_CONFIGS: + preset = PMF_PRESET_CONFIGS[model_type] + sample_size = int(preset["sample_size"]) + patch_size = int(preset["patch_size"]) + hidden_size = int(preset["hidden_size"]) + depth = int(preset["depth"]) + num_attention_heads = int(preset["num_attention_heads"]) + bottleneck_dim = int(preset["bottleneck_dim"]) + aux_head_depth = int(preset["aux_head_depth"]) + + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.depth = depth + self.num_attention_heads = num_attention_heads + self.aux_head_depth = aux_head_depth + self.num_class_tokens = num_class_tokens + self.num_time_tokens = num_time_tokens + self.num_cfg_tokens = num_cfg_tokens + self.num_interval_tokens = num_interval_tokens + self.prefix_tokens = ( + num_class_tokens + num_cfg_tokens + 2 * num_interval_tokens + num_time_tokens + ) + self.t_clip_min = t_clip_min + self.eval_mode = eval_mode + self.gradient_checkpointing = False + + self.x_embedder = PMFBottleneckPatchEmbedder( + sample_size, + patch_size, + bottleneck_dim, + in_channels, + hidden_size, + bias=True, + ) + embed_kwargs = dict(hidden_size=hidden_size, init_constant=embedding_init_constant) + self.h_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.omega_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_start_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_end_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.y_embedder = PMFLabelEmbedder(num_classes, hidden_size, init_constant=embedding_init_constant) + + token_std = token_init_constant / math.sqrt(hidden_size) + self.time_tokens = nn.Parameter(torch.randn(1, num_time_tokens, hidden_size) * token_std) + self.class_tokens = nn.Parameter(torch.randn(1, num_class_tokens, hidden_size) * token_std) + self.omega_tokens = nn.Parameter(torch.randn(1, num_cfg_tokens, hidden_size) * token_std) + self.t_min_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + self.t_max_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + + total_tokens = self.x_embedder.num_patches + self.prefix_tokens + self.pos_embed = nn.Parameter(torch.randn(1, total_tokens, hidden_size) * 0.02) + + head_dim = hidden_size // num_attention_heads + self.register_buffer( + "rope_freqs", + precompute_rope_freqs(head_dim, self.x_embedder.num_patches), + persistent=False, + ) + + shared_depth = depth - aux_head_depth + block_kwargs = dict( + hidden_size=hidden_size, + num_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + weight_init_constant=weight_init_constant, + eps=norm_eps, + ) + self.shared_blocks = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(shared_depth)]) + self.u_heads = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth)]) + self.v_heads = nn.ModuleList( + [PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth if not eval_mode else 0)] + ) + self.u_final_layer = PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + self.v_final_layer = ( + PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + if not eval_mode + else None + ) + + def _build_sequence( + self, + sample: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + t_min: torch.Tensor, + t_max: torch.Tensor, + class_labels: torch.Tensor, + ) -> torch.Tensor: + x_embed = self.x_embedder(sample) + h_embed = self.h_embedder(h) + omega_embed = self.omega_embedder(1 - 1 / omega) + t_min_embed = self.cfg_t_start_embedder(t_min) + t_max_embed = self.cfg_t_end_embedder(t_max) + y_embed = self.y_embedder(class_labels) + + time_tokens = self.time_tokens + h_embed.unsqueeze(1) + omega_tokens = self.omega_tokens + omega_embed.unsqueeze(1) + t_min_tokens = self.t_min_tokens + t_min_embed.unsqueeze(1) + t_max_tokens = self.t_max_tokens + t_max_embed.unsqueeze(1) + class_tokens = self.class_tokens + y_embed.unsqueeze(1) + + seq = torch.cat( + [class_tokens, omega_tokens, t_min_tokens, t_max_tokens, time_tokens, x_embed], + dim=1, + ) + return seq + self.pos_embed + + def _unpatchify(self, tokens: torch.Tensor) -> torch.Tensor: + batch_size = tokens.shape[0] + patch = self.patch_size + grid = int(tokens.shape[1] ** 0.5) + channels = self.out_channels + x = tokens.reshape(batch_size, grid, grid, patch, patch, channels) + x = torch.einsum("nhwpqc->nchpwq", x) + return x.reshape(batch_size, channels, grid * patch, grid * patch) + + def forward( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: Optional[torch.Tensor] = None, + omega: Optional[torch.Tensor] = None, + guidance_interval_min: Optional[torch.Tensor] = None, + guidance_interval_max: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> PMFTransformer2DOutput | Tuple[torch.Tensor, Optional[torch.Tensor]]: + batch_size = sample.shape[0] + timestep = self._expand_batch(timestep, batch_size, sample.device, sample.dtype) + h = self._expand_batch(h if h is not None else timestep, batch_size, sample.device, sample.dtype) + omega = self._expand_batch( + omega if omega is not None else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_min = self._expand_batch( + guidance_interval_min + if guidance_interval_min is not None + else torch.zeros(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_max = self._expand_batch( + guidance_interval_max + if guidance_interval_max is not None + else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + + seq = self._build_sequence(sample, h, omega, guidance_interval_min, guidance_interval_max, class_labels) + rope_freqs = self.rope_freqs.to(device=sample.device) + + for block in self.shared_blocks: + if self.training and self.gradient_checkpointing: + seq = torch.utils.checkpoint.checkpoint(block, seq, rope_freqs, use_reentrant=False) + else: + seq = block(seq, rope_freqs) + + u_seq = v_seq = seq + for block in self.u_heads: + if self.training and self.gradient_checkpointing: + u_seq = torch.utils.checkpoint.checkpoint(block, u_seq, rope_freqs, use_reentrant=False) + else: + u_seq = block(u_seq, rope_freqs) + + for block in self.v_heads: + if self.training and self.gradient_checkpointing: + v_seq = torch.utils.checkpoint.checkpoint(block, v_seq, rope_freqs, use_reentrant=False) + else: + v_seq = block(v_seq, rope_freqs) + + u_tokens = u_seq[:, self.prefix_tokens :] + u_pred = self._unpatchify(self.u_final_layer(u_tokens)) + t = timestep.reshape(batch_size, 1, 1, 1) + u = (sample - u_pred) / torch.clamp(t, min=self.t_clip_min) + + v = None + if self.v_final_layer is not None: + v_tokens = v_seq[:, self.prefix_tokens :] + v_pred = self._unpatchify(self.v_final_layer(v_tokens)) + v = (sample - v_pred) / torch.clamp(t, min=self.t_clip_min) + + if not return_dict: + return (u, v) + return PMFTransformer2DOutput(u=u, v=v) + + @staticmethod + def _expand_batch( + value: torch.Tensor, + batch_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + value = torch.as_tensor(value, device=device, dtype=dtype) + if value.ndim == 0: + value = value.reshape(1) + if value.shape[0] == 1 and batch_size > 1: + value = value.expand(batch_size) + return value.reshape(batch_size) + + @classmethod + def from_pmf_checkpoint( + cls, + checkpoint_path: str, + model_type: str | None = None, + map_location: str = "cpu", + strict: bool = False, + ) -> Tuple["PMFTransformer2DModel", Dict[str, object]]: + checkpoint = torch.load(checkpoint_path, map_location=map_location, weights_only=False) + if isinstance(checkpoint, dict) and "state_dict" in checkpoint: + state_dict = checkpoint["state_dict"] + else: + state_dict = checkpoint + + if model_type is None: + for key in ("model_type", "model_str", "model"): + if isinstance(checkpoint, dict) and key in checkpoint: + model_type = checkpoint[key] + break + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type is None: + raise ValueError("model_type is required when it cannot be inferred from the checkpoint.") + + config = dict(PMF_PRESET_CONFIGS[model_type]) + config["model_type"] = model_type + config["eval_mode"] = True + model = cls(**config) + model.load_state_dict(remap_legacy_state_dict(state_dict), strict=strict) + metadata = {"checkpoint_path": checkpoint_path, "model_type": model_type} + return model, metadata + + def to_pmf_checkpoint(self, prefix: str = "net.") -> Dict[str, torch.Tensor]: + state_dict: Dict[str, torch.Tensor] = {} + for key, value in self.state_dict().items(): + state_dict[f"{prefix}{key}"] = value.detach().cpu() + return state_dict + + @property + def net(self): + return self + + +PMFDiffusersModel = PMFTransformer2DModel diff --git a/pMF-H-16-SIM/README.md b/pMF-H-16-SIM/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3a4edce1ffc590117ef4eea0219aab1b625dd559 --- /dev/null +++ b/pMF-H-16-SIM/README.md @@ -0,0 +1,44 @@ +--- +license: mit +library_name: diffusers +pipeline_tag: text-to-image +tags: +- diffusers +- pmf +- image-generation +- class-conditional +- imagenet +inference: true +--- + +# pMF-H-16-SIM + +Self-contained Diffusers variant for **pMF-H/16 (FD-SIM post-trained)** (Pixel Mean Flows). + +Recommended settings: `guidance_scale=7.0`, interval `[0.2, 0.6]`, `noise_scale=2.0`. + +## Load + +```python +from pathlib import Path +from diffusers import DiffusionPipeline +import torch + +model_dir = Path("./pMF-H-16-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + local_files_only=True, + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.bfloat16, +).to("cuda") + +image = pipe( + class_labels=207, + num_inference_steps=1, + guidance_scale=7.0, + guidance_interval_min=0.2, + guidance_interval_max=0.6, + noise_scale=2.0, +).images[0] +``` diff --git a/pMF-H-16-SIM/model_index.json b/pMF-H-16-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..ec00b5bd38c72b4310e6e3cb2c65ce31ef2427b4 --- /dev/null +++ b/pMF-H-16-SIM/model_index.json @@ -0,0 +1,1017 @@ +{ + "_class_name": [ + "pipeline", + "PMFPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_pmf", + "PMFScheduler" + ], + "transformer": [ + "transformer_pmf", + "PMFTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + } +} diff --git a/pMF-H-16-SIM/pipeline.py b/pMF-H-16-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..314f675b581fe55ede8cbc0d1f2e1f850e5e9f34 --- /dev/null +++ b/pMF-H-16-SIM/pipeline.py @@ -0,0 +1,289 @@ +"""Hub custom pipeline: PMFPipeline.""" + +from __future__ import annotations + +import inspect +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +DEFAULT_CFG_BY_MODEL: Dict[str, Dict[str, float]] = { + "pMF-B/16": {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8}, + "pMF-B/32": {"guidance_scale": 6.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.7}, + "pMF-L/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.7}, + "pMF-L/32": {"guidance_scale": 7.5, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/32": {"guidance_scale": 5.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.6}, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + + +class PMFPipeline(DiffusionPipeline): + model_cpu_offload_seq = "transformer" + + def __init__( + self, + transformer: Any, + scheduler: Any, + id2label: Optional[Dict[Union[int, str], str]] = None, + ) -> None: + super().__init__() + if scheduler is None: + raise ValueError("PMFPipeline requires a scheduler loaded from the checkpoint.") + self.register_modules(transformer=transformer, scheduler=scheduler) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + + @staticmethod + def prepare_extra_step_kwargs( + scheduler: Any, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + eta: float | None = None, + ) -> Dict[str, Any]: + kwargs: Dict[str, Any] = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + @staticmethod + def _prepare_generator( + generator: Optional[Union[torch.Generator, List[torch.Generator]]], + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + """Reset generator state so repeated `__call__` with the same object is reproducible.""" + if generator is None: + return None + if isinstance(generator, list): + for gen in generator: + if gen is not None: + gen.manual_seed(int(gen.initial_seed())) + return generator + generator.manual_seed(int(generator.initial_seed())) + return generator + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + loaded = self._read_id2label_from_model_index(getattr(self.config, "_name_or_path", None)) + if loaded: + self._id2label = loaded + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _read_id2label_from_model_index(variant_path: Optional[str]) -> Dict[int, str]: + if not variant_path: + return {} + model_index_path = Path(variant_path).resolve() / "model_index.json" + if not model_index_path.exists(): + return {} + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if not isinstance(id2label, dict): + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + self._ensure_labels_loaded() + if not self.labels: + raise ValueError("No labels loaded. Ensure `id2label` exists in model_index.json.") + labels = [label] if isinstance(label, str) else label + missing = [item for item in labels if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Example valid labels: {preview}, ...") + return [self.labels[item] for item in labels] + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) # type: ignore[arg-type] + return [int(class_id) for class_id in class_labels] + + def _recommended_noise_scale(self) -> float: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in RECOMMENDED_NOISE_BY_MODEL: + return RECOMMENDED_NOISE_BY_MODEL[model_type] + image_size = int(self.transformer.config.sample_size) + return {256: 1.0, 512: 2.0}.get(image_size, 1.0) + + def _default_cfg(self) -> Dict[str, float]: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in DEFAULT_CFG_BY_MODEL: + return dict(DEFAULT_CFG_BY_MODEL[model_type]) + return {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8} + + def predict_u( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + guidance_interval_min: torch.Tensor, + guidance_interval_max: torch.Tensor, + ) -> torch.Tensor: + output = self.transformer( + sample=sample, + timestep=timestep, + class_labels=class_labels, + h=h, + omega=omega, + guidance_interval_min=guidance_interval_min, + guidance_interval_max=guidance_interval_max, + return_dict=True, + ) + return output.u + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + num_inference_steps: int = 1, + guidance_scale: Optional[float] = None, + guidance_interval_min: Optional[float] = None, + guidance_interval_max: Optional[float] = None, + noise_scale: Optional[float] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + if output_type not in {"pil", "np", "pt"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt'.") + + defaults = self._default_cfg() + if guidance_scale is None: + guidance_scale = defaults["guidance_scale"] + if guidance_interval_min is None: + guidance_interval_min = defaults["guidance_interval_min"] + if guidance_interval_max is None: + guidance_interval_max = defaults["guidance_interval_max"] + if noise_scale is None: + noise_scale = self._recommended_noise_scale() + + class_label_ids = self._normalize_class_labels(class_labels) + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + generator = self._prepare_generator(generator) + image_size = int(self.transformer.config.sample_size) + channels = int(self.transformer.config.in_channels) + null_class_val = int( + getattr(self.transformer.config, "num_classes", getattr(self.transformer.config, "num_class_embeds", 1000)) + ) + + latents = randn_tensor( + shape=(batch_size, channels, image_size, image_size), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) * noise_scale + + class_labels_t = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1) + class_labels_t = class_labels_t.clamp(0, null_class_val - 1) + + device = latents.device + dtype = latents.dtype + omega = torch.full((batch_size,), guidance_scale, device=device, dtype=dtype) + t_min = torch.full((batch_size,), guidance_interval_min, device=device, dtype=dtype) + t_max = torch.full((batch_size,), guidance_interval_max, device=device, dtype=dtype) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + timesteps = self.scheduler.timesteps + + for step_index in self.progress_bar(range(num_inference_steps)): + t = timesteps[step_index] + t_next = timesteps[step_index + 1] + h = (t - t_next).expand(batch_size).to(device=device, dtype=dtype) + t_batch = t.expand(batch_size).to(device=device, dtype=dtype) + + u = self.predict_u( + sample=latents, + timestep=t_batch, + class_labels=class_labels_t, + h=h, + omega=omega, + guidance_interval_min=t_min, + guidance_interval_max=t_max, + ) + latents = self.scheduler.step(u, t, latents, **extra_step_kwargs).prev_sample + + images_pt = ((latents.float().clamp(-1, 1) + 1.0) / 2.0).cpu() + if output_type == "pt": + images = images_pt + elif output_type == "np": + images = images_pt.permute(0, 2, 3, 1).numpy() + else: + images = self.numpy_to_pil(images_pt.permute(0, 2, 3, 1).numpy()) + + self.maybe_free_model_hooks() + + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) + + +PMFPipelineOutput = ImagePipelineOutput + +__all__ = ["PMFPipeline", "PMFPipelineOutput"] diff --git a/pMF-H-16-SIM/scheduler/scheduler_config.json b/pMF-H-16-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..341ff049a80407b660e0c785f274b39768b8ebce --- /dev/null +++ b/pMF-H-16-SIM/scheduler/scheduler_config.json @@ -0,0 +1,6 @@ +{ + "_class_name": "PMFScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000, + "stochastic_sampling": false +} diff --git a/pMF-H-16-SIM/scheduler/scheduling_pmf.py b/pMF-H-16-SIM/scheduler/scheduling_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..e4a599c65bb4c64bb5296ab66e5495ac0d936139 --- /dev/null +++ b/pMF-H-16-SIM/scheduler/scheduling_pmf.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class PMFSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class PMFScheduler(SchedulerMixin, ConfigMixin): + """Scheduler for Pixel Mean Flow sampling (t: 1 -> 0).""" + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + stochastic_sampling: bool = False, + ): + del num_train_timesteps + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> None: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.flatten()[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + @staticmethod + def _expand_time(value: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + value = value.to(dtype=sample.dtype, device=sample.device) + while value.ndim < sample.ndim: + value = value.unsqueeze(-1) + return value + + def _sample_noise( + self, + sample: torch.Tensor, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> torch.Tensor: + return randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[PMFSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = self._expand_time(t - t_next, sample) + + if self.config.stochastic_sampling: + t_expanded = self._expand_time(t, sample) + t_next_expanded = self._expand_time(t_next, sample) + x0 = sample - t_expanded * model_output + noise = self._sample_noise(sample, generator=generator) + prev_sample = (1.0 - t_next_expanded) * x0 + t_next_expanded * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return PMFSchedulerOutput(prev_sample=prev_sample) diff --git a/pMF-H-16-SIM/transformer/config.json b/pMF-H-16-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d6cd813d249d489ddabe033abe225cf23e635a50 --- /dev/null +++ b/pMF-H-16-SIM/transformer/config.json @@ -0,0 +1,26 @@ +{ + "_class_name": "PMFTransformer2DModel", + "_diffusers_version": "0.38.0", + "aux_head_depth": 8, + "bottleneck_dim": 256, + "depth": 48, + "embedding_init_constant": 1.0, + "eval_mode": true, + "hidden_size": 1280, + "in_channels": 3, + "mlp_ratio": 2.6666666666666665, + "model_type": "pMF-H/16", + "norm_eps": 1e-06, + "num_attention_heads": 16, + "num_cfg_tokens": 4, + "num_class_embeds": null, + "num_class_tokens": 8, + "num_classes": 1000, + "num_interval_tokens": 2, + "num_time_tokens": 4, + "patch_size": 16, + "sample_size": 256, + "t_clip_min": 0.05, + "token_init_constant": 1.0, + "weight_init_constant": 0.32 +} diff --git a/pMF-H-16-SIM/transformer/diffusion_pytorch_model.safetensors b/pMF-H-16-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..27b8a56889d6e0e5ea84eacc3b9b10ffb4226c79 --- /dev/null +++ b/pMF-H-16-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4a7c5cf93e89601e79c9472c49ab6e31907860d4cc630aedc34ad36976b597a +size 3822104552 diff --git a/pMF-H-16-SIM/transformer/transformer_pmf.py b/pMF-H-16-SIM/transformer/transformer_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..623b1c9b9cd2d9dc8833056f6776087a85a602a1 --- /dev/null +++ b/pMF-H-16-SIM/transformer/transformer_pmf.py @@ -0,0 +1,664 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from math import sqrt +from typing import Dict, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm +from diffusers.utils import BaseOutput + + +PMF_PRESET_CONFIGS: Dict[str, Dict[str, object]] = { + "pMF-B/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-B/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-H/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, + "pMF-H/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + +# Legacy torch repo keys (pmfDiT_*) +LEGACY_MODEL_ALIASES: Dict[str, str] = { + "pmfDiT_B_16": "pMF-B/16", + "pmfDiT_B_32": "pMF-B/32", + "pmfDiT_L_16": "pMF-L/16", + "pmfDiT_L_32": "pMF-L/32", + "pmfDiT_H_16": "pMF-H/16", + "pmfDiT_H_32": "pMF-H/32", +} + + +@dataclass +class PMFTransformer2DOutput(BaseOutput): + u: torch.Tensor + v: Optional[torch.Tensor] = None + + +def remap_legacy_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Map wrapper/backbone keys from legacy checkpoints to native PMFTransformer2DModel keys.""" + remapped: Dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + for prefix in ("transformer.", "net."): + if new_key.startswith(prefix): + new_key = new_key[len(prefix) :] + break + # Official PyTorch checkpoints use TorchLinear/TorchEmbedding wrappers. + new_key = new_key.replace("._flax_linear", "").replace("._flax_embedding", "") + if new_key == "rope_freqs": + continue + remapped[new_key] = value + return remapped + + +def config_from_legacy(config: Dict[str, object]) -> Dict[str, object]: + """Build native config kwargs from a legacy config.json dict.""" + model_type = config.get("model_type") or config.get("model_name") or config.get("model_str") + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type not in PMF_PRESET_CONFIGS: + raise ValueError(f"Unknown pMF preset '{model_type}'. Known: {list(PMF_PRESET_CONFIGS)}") + + preset = dict(PMF_PRESET_CONFIGS[model_type]) + preset["num_classes"] = int(config.get("num_class_embeds") or config.get("num_classes") or 1000) + preset["model_type"] = model_type + if config.get("sample_size") is not None: + preset["sample_size"] = int(config["sample_size"]) + if config.get("eval_mode") is not None: + preset["eval_mode"] = bool(config["eval_mode"]) + return preset + + +def _scaled_linear( + in_features: int, + out_features: int, + *, + bias: bool = True, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + bias_init: str = "zeros", +) -> nn.Linear: + layer = nn.Linear(in_features, out_features, bias=bias) + if weight_init == "scaled_variance": + std = init_constant / sqrt(in_features) + nn.init.normal_(layer.weight, std=std) + elif weight_init == "zeros": + nn.init.zeros_(layer.weight) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + + if bias: + if bias_init == "zeros": + nn.init.zeros_(layer.bias) + else: + raise ValueError(f"Invalid bias_init: {bias_init}") + return layer + + +class PMFTimestepEmbedder(nn.Module): + def __init__( + self, + hidden_size: int, + frequency_embedding_size: int = 256, + init_constant: float = 1.0, + ): + super().__init__() + init_kwargs = dict( + out_features=hidden_size, + bias=True, + weight_init="scaled_variance", + init_constant=init_constant, + bias_init="zeros", + ) + self.mlp = nn.Sequential( + _scaled_linear(frequency_embedding_size, **init_kwargs), + nn.SiLU(), + _scaled_linear(hidden_size, **init_kwargs), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=torch.float32, device=t.device) + / half + ) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + return self.mlp(t_freq) + + +class PMFLabelEmbedder(nn.Module): + def __init__(self, num_classes: int, hidden_size: int, init_constant: float = 1.0): + super().__init__() + self.embedding_table = nn.Embedding(num_classes + 1, hidden_size) + nn.init.normal_(self.embedding_table.weight, std=init_constant / sqrt(hidden_size)) + + def forward(self, labels: torch.Tensor) -> torch.Tensor: + return self.embedding_table(labels) + + +class PMFBottleneckPatchEmbedder(nn.Module): + def __init__( + self, + input_size: int, + patch_size: int, + pca_channels: int, + in_channels: int, + hidden_size: int, + bias: bool = True, + ): + super().__init__() + self.patch_size = (patch_size, patch_size) + self.num_patches = (input_size // patch_size) ** 2 + self.proj1 = nn.Conv2d( + in_channels, + pca_channels, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + ) + self.proj2 = nn.Conv2d(pca_channels, hidden_size, kernel_size=1, stride=1, bias=bias) + + kh = kw = patch_size + fan_in = kh * kw * in_channels + fan_out = pca_channels + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj1.weight, -limit, limit) + fan_in = pca_channels + fan_out = hidden_size + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj2.weight, -limit, limit) + if bias: + nn.init.zeros_(self.proj1.bias) + nn.init.zeros_(self.proj2.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj2(self.proj1(x)) + return x.flatten(2).transpose(1, 2) + + +def precompute_rope_freqs(dim: int, seq_len: int, theta: float = 10000.0) -> torch.Tensor: + dim = dim // 2 + grid_size = int(seq_len**0.5) + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + positions = torch.arange(grid_size, dtype=torch.float32) + freqs_h = torch.einsum("i,j->ij", positions, freqs) + freqs_w = torch.einsum("i,j->ij", positions, freqs) + freqs_2d = torch.cat( + [ + torch.tile(freqs_h[:, None, :], (1, grid_size, 1)), + torch.tile(freqs_w[None, :, :], (grid_size, 1, 1)), + ], + dim=-1, + ) + real = torch.cos(freqs_2d).reshape(seq_len, dim) + imag = torch.sin(freqs_2d).reshape(seq_len, dim) + return torch.complex(real, imag) + + +def apply_rotary_pos_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + x_float = x.to(torch.float32) + x_complex = torch.view_as_complex(x_float.reshape(*x_float.shape[:-1], -1, 2).contiguous()) + freqs_cis = freqs_cis.unsqueeze(0).unsqueeze(2) + token_count = freqs_cis.shape[1] + x_rotated = x_complex.clone() + x_rotated[:, -token_count:, :] = x_complex[:, -token_count:, :] * freqs_cis + x_out = torch.view_as_real(x_rotated).flatten(-2) + return x_out.to(x.dtype) + + +class PMFAttention(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + init_kwargs = dict( + bias=False, + weight_init="scaled_variance", + init_constant=weight_init_constant, + ) + self.q_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.k_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.v_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.out_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.q_norm = RMSNorm(self.head_dim, eps=eps) + self.k_norm = RMSNorm(self.head_dim, eps=eps) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + batch_size, seq_len, channels = x.shape + q = self.q_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + k = self.k_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + v = self.v_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + + q = self.q_norm(q) + k = self.k_norm(k) + q = apply_rotary_pos_emb(q, rope_freqs) + k = apply_rotary_pos_emb(k, rope_freqs) + + query = q / math.sqrt(self.head_dim) + attn_weights = torch.einsum("bqhd,bkhd->bhqk", query, k) + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32) + attn = torch.einsum("bhqk,bkhd->bqhd", attn_weights, v) + attn = attn.reshape(batch_size, seq_len, channels) + return self.out_proj(attn) + + +class PMFSwiGLUMlp(nn.Module): + def __init__(self, dim: int, hidden_dim: int, weight_init_constant: float = 0.32): + super().__init__() + init_kwargs = dict(bias=False, weight_init="scaled_variance", init_constant=weight_init_constant) + self.w1 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w3 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w2 = _scaled_linear(hidden_dim, dim, **init_kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + + +class PMFTransformerBlock(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float = 8 / 3, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.norm1 = RMSNorm(hidden_size, eps=eps) + self.attn = PMFAttention(hidden_size, num_heads, weight_init_constant=weight_init_constant, eps=eps) + self.norm2 = RMSNorm(hidden_size, eps=eps) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + if hidden_size > 1024: + mlp_hidden_dim = (mlp_hidden_dim + 7) // 8 * 8 + self.mlp = PMFSwiGLUMlp(hidden_size, mlp_hidden_dim, weight_init_constant=weight_init_constant) + self.attn_scale = nn.Parameter(torch.zeros(hidden_size)) + self.mlp_scale = nn.Parameter(torch.zeros(hidden_size)) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), rope_freqs) * self.attn_scale + x = x + self.mlp(self.norm2(x)) * self.mlp_scale + return x + + +class PMFFinalLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int, eps: float = 1e-6): + super().__init__() + self.norm = RMSNorm(hidden_size, eps=eps) + self.linear = _scaled_linear( + hidden_size, + patch_size * patch_size * out_channels, + bias=True, + weight_init="zeros", + bias_init="zeros", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(self.norm(x)) + + +class PMFTransformer2DModel(ModelMixin, ConfigMixin): + """Native diffusers implementation of the pMF DiT backbone.""" + + _supports_gradient_checkpointing = True + _skip_layerwise_casting_patterns = ["pos_embed", "rope_freqs"] + + @register_to_config + def __init__( + self, + sample_size: int = 256, + patch_size: int = 16, + in_channels: int = 3, + hidden_size: int = 768, + depth: int = 16, + num_attention_heads: int = 12, + mlp_ratio: float = 8 / 3, + num_classes: int = 1000, + bottleneck_dim: int = 128, + aux_head_depth: int = 8, + num_class_tokens: int = 8, + num_time_tokens: int = 4, + num_cfg_tokens: int = 4, + num_interval_tokens: int = 2, + token_init_constant: float = 1.0, + embedding_init_constant: float = 1.0, + weight_init_constant: float = 0.32, + eval_mode: bool = True, + model_type: str | None = None, + num_class_embeds: int | None = None, + t_clip_min: float = 0.05, + norm_eps: float = 1e-6, + ): + super().__init__() + if num_class_embeds is not None: + num_classes = int(num_class_embeds) + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type in PMF_PRESET_CONFIGS: + preset = PMF_PRESET_CONFIGS[model_type] + sample_size = int(preset["sample_size"]) + patch_size = int(preset["patch_size"]) + hidden_size = int(preset["hidden_size"]) + depth = int(preset["depth"]) + num_attention_heads = int(preset["num_attention_heads"]) + bottleneck_dim = int(preset["bottleneck_dim"]) + aux_head_depth = int(preset["aux_head_depth"]) + + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.depth = depth + self.num_attention_heads = num_attention_heads + self.aux_head_depth = aux_head_depth + self.num_class_tokens = num_class_tokens + self.num_time_tokens = num_time_tokens + self.num_cfg_tokens = num_cfg_tokens + self.num_interval_tokens = num_interval_tokens + self.prefix_tokens = ( + num_class_tokens + num_cfg_tokens + 2 * num_interval_tokens + num_time_tokens + ) + self.t_clip_min = t_clip_min + self.eval_mode = eval_mode + self.gradient_checkpointing = False + + self.x_embedder = PMFBottleneckPatchEmbedder( + sample_size, + patch_size, + bottleneck_dim, + in_channels, + hidden_size, + bias=True, + ) + embed_kwargs = dict(hidden_size=hidden_size, init_constant=embedding_init_constant) + self.h_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.omega_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_start_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_end_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.y_embedder = PMFLabelEmbedder(num_classes, hidden_size, init_constant=embedding_init_constant) + + token_std = token_init_constant / math.sqrt(hidden_size) + self.time_tokens = nn.Parameter(torch.randn(1, num_time_tokens, hidden_size) * token_std) + self.class_tokens = nn.Parameter(torch.randn(1, num_class_tokens, hidden_size) * token_std) + self.omega_tokens = nn.Parameter(torch.randn(1, num_cfg_tokens, hidden_size) * token_std) + self.t_min_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + self.t_max_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + + total_tokens = self.x_embedder.num_patches + self.prefix_tokens + self.pos_embed = nn.Parameter(torch.randn(1, total_tokens, hidden_size) * 0.02) + + head_dim = hidden_size // num_attention_heads + self.register_buffer( + "rope_freqs", + precompute_rope_freqs(head_dim, self.x_embedder.num_patches), + persistent=False, + ) + + shared_depth = depth - aux_head_depth + block_kwargs = dict( + hidden_size=hidden_size, + num_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + weight_init_constant=weight_init_constant, + eps=norm_eps, + ) + self.shared_blocks = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(shared_depth)]) + self.u_heads = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth)]) + self.v_heads = nn.ModuleList( + [PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth if not eval_mode else 0)] + ) + self.u_final_layer = PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + self.v_final_layer = ( + PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + if not eval_mode + else None + ) + + def _build_sequence( + self, + sample: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + t_min: torch.Tensor, + t_max: torch.Tensor, + class_labels: torch.Tensor, + ) -> torch.Tensor: + x_embed = self.x_embedder(sample) + h_embed = self.h_embedder(h) + omega_embed = self.omega_embedder(1 - 1 / omega) + t_min_embed = self.cfg_t_start_embedder(t_min) + t_max_embed = self.cfg_t_end_embedder(t_max) + y_embed = self.y_embedder(class_labels) + + time_tokens = self.time_tokens + h_embed.unsqueeze(1) + omega_tokens = self.omega_tokens + omega_embed.unsqueeze(1) + t_min_tokens = self.t_min_tokens + t_min_embed.unsqueeze(1) + t_max_tokens = self.t_max_tokens + t_max_embed.unsqueeze(1) + class_tokens = self.class_tokens + y_embed.unsqueeze(1) + + seq = torch.cat( + [class_tokens, omega_tokens, t_min_tokens, t_max_tokens, time_tokens, x_embed], + dim=1, + ) + return seq + self.pos_embed + + def _unpatchify(self, tokens: torch.Tensor) -> torch.Tensor: + batch_size = tokens.shape[0] + patch = self.patch_size + grid = int(tokens.shape[1] ** 0.5) + channels = self.out_channels + x = tokens.reshape(batch_size, grid, grid, patch, patch, channels) + x = torch.einsum("nhwpqc->nchpwq", x) + return x.reshape(batch_size, channels, grid * patch, grid * patch) + + def forward( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: Optional[torch.Tensor] = None, + omega: Optional[torch.Tensor] = None, + guidance_interval_min: Optional[torch.Tensor] = None, + guidance_interval_max: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> PMFTransformer2DOutput | Tuple[torch.Tensor, Optional[torch.Tensor]]: + batch_size = sample.shape[0] + timestep = self._expand_batch(timestep, batch_size, sample.device, sample.dtype) + h = self._expand_batch(h if h is not None else timestep, batch_size, sample.device, sample.dtype) + omega = self._expand_batch( + omega if omega is not None else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_min = self._expand_batch( + guidance_interval_min + if guidance_interval_min is not None + else torch.zeros(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_max = self._expand_batch( + guidance_interval_max + if guidance_interval_max is not None + else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + + seq = self._build_sequence(sample, h, omega, guidance_interval_min, guidance_interval_max, class_labels) + rope_freqs = self.rope_freqs.to(device=sample.device) + + for block in self.shared_blocks: + if self.training and self.gradient_checkpointing: + seq = torch.utils.checkpoint.checkpoint(block, seq, rope_freqs, use_reentrant=False) + else: + seq = block(seq, rope_freqs) + + u_seq = v_seq = seq + for block in self.u_heads: + if self.training and self.gradient_checkpointing: + u_seq = torch.utils.checkpoint.checkpoint(block, u_seq, rope_freqs, use_reentrant=False) + else: + u_seq = block(u_seq, rope_freqs) + + for block in self.v_heads: + if self.training and self.gradient_checkpointing: + v_seq = torch.utils.checkpoint.checkpoint(block, v_seq, rope_freqs, use_reentrant=False) + else: + v_seq = block(v_seq, rope_freqs) + + u_tokens = u_seq[:, self.prefix_tokens :] + u_pred = self._unpatchify(self.u_final_layer(u_tokens)) + t = timestep.reshape(batch_size, 1, 1, 1) + u = (sample - u_pred) / torch.clamp(t, min=self.t_clip_min) + + v = None + if self.v_final_layer is not None: + v_tokens = v_seq[:, self.prefix_tokens :] + v_pred = self._unpatchify(self.v_final_layer(v_tokens)) + v = (sample - v_pred) / torch.clamp(t, min=self.t_clip_min) + + if not return_dict: + return (u, v) + return PMFTransformer2DOutput(u=u, v=v) + + @staticmethod + def _expand_batch( + value: torch.Tensor, + batch_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + value = torch.as_tensor(value, device=device, dtype=dtype) + if value.ndim == 0: + value = value.reshape(1) + if value.shape[0] == 1 and batch_size > 1: + value = value.expand(batch_size) + return value.reshape(batch_size) + + @classmethod + def from_pmf_checkpoint( + cls, + checkpoint_path: str, + model_type: str | None = None, + map_location: str = "cpu", + strict: bool = False, + ) -> Tuple["PMFTransformer2DModel", Dict[str, object]]: + checkpoint = torch.load(checkpoint_path, map_location=map_location, weights_only=False) + if isinstance(checkpoint, dict) and "state_dict" in checkpoint: + state_dict = checkpoint["state_dict"] + else: + state_dict = checkpoint + + if model_type is None: + for key in ("model_type", "model_str", "model"): + if isinstance(checkpoint, dict) and key in checkpoint: + model_type = checkpoint[key] + break + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type is None: + raise ValueError("model_type is required when it cannot be inferred from the checkpoint.") + + config = dict(PMF_PRESET_CONFIGS[model_type]) + config["model_type"] = model_type + config["eval_mode"] = True + model = cls(**config) + model.load_state_dict(remap_legacy_state_dict(state_dict), strict=strict) + metadata = {"checkpoint_path": checkpoint_path, "model_type": model_type} + return model, metadata + + def to_pmf_checkpoint(self, prefix: str = "net.") -> Dict[str, torch.Tensor]: + state_dict: Dict[str, torch.Tensor] = {} + for key, value in self.state_dict().items(): + state_dict[f"{prefix}{key}"] = value.detach().cpu() + return state_dict + + @property + def net(self): + return self + + +PMFDiffusersModel = PMFTransformer2DModel diff --git a/pMF-H-32-SIM/README.md b/pMF-H-32-SIM/README.md new file mode 100644 index 0000000000000000000000000000000000000000..20ae306d4b83bacfef6fc81d23b300c2f069da73 --- /dev/null +++ b/pMF-H-32-SIM/README.md @@ -0,0 +1,44 @@ +--- +license: mit +library_name: diffusers +pipeline_tag: text-to-image +tags: +- diffusers +- pmf +- image-generation +- class-conditional +- imagenet +inference: true +--- + +# pMF-H-32-SIM + +Self-contained Diffusers variant for **pMF-H/32 (FD-SIM post-trained)** (Pixel Mean Flows). + +Recommended settings: `guidance_scale=5.5`, interval `[0.1, 0.6]`, `noise_scale=4.0`. + +## Load + +```python +from pathlib import Path +from diffusers import DiffusionPipeline +import torch + +model_dir = Path("./pMF-H-32-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + local_files_only=True, + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.bfloat16, +).to("cuda") + +image = pipe( + class_labels=207, + num_inference_steps=1, + guidance_scale=5.5, + guidance_interval_min=0.1, + guidance_interval_max=0.6, + noise_scale=4.0, +).images[0] +``` diff --git a/pMF-H-32-SIM/model_index.json b/pMF-H-32-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..ec00b5bd38c72b4310e6e3cb2c65ce31ef2427b4 --- /dev/null +++ b/pMF-H-32-SIM/model_index.json @@ -0,0 +1,1017 @@ +{ + "_class_name": [ + "pipeline", + "PMFPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_pmf", + "PMFScheduler" + ], + "transformer": [ + "transformer_pmf", + "PMFTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + } +} diff --git a/pMF-H-32-SIM/pipeline.py b/pMF-H-32-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..314f675b581fe55ede8cbc0d1f2e1f850e5e9f34 --- /dev/null +++ b/pMF-H-32-SIM/pipeline.py @@ -0,0 +1,289 @@ +"""Hub custom pipeline: PMFPipeline.""" + +from __future__ import annotations + +import inspect +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +DEFAULT_CFG_BY_MODEL: Dict[str, Dict[str, float]] = { + "pMF-B/16": {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8}, + "pMF-B/32": {"guidance_scale": 6.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.7}, + "pMF-L/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.7}, + "pMF-L/32": {"guidance_scale": 7.5, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/32": {"guidance_scale": 5.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.6}, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + + +class PMFPipeline(DiffusionPipeline): + model_cpu_offload_seq = "transformer" + + def __init__( + self, + transformer: Any, + scheduler: Any, + id2label: Optional[Dict[Union[int, str], str]] = None, + ) -> None: + super().__init__() + if scheduler is None: + raise ValueError("PMFPipeline requires a scheduler loaded from the checkpoint.") + self.register_modules(transformer=transformer, scheduler=scheduler) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + + @staticmethod + def prepare_extra_step_kwargs( + scheduler: Any, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + eta: float | None = None, + ) -> Dict[str, Any]: + kwargs: Dict[str, Any] = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + @staticmethod + def _prepare_generator( + generator: Optional[Union[torch.Generator, List[torch.Generator]]], + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + """Reset generator state so repeated `__call__` with the same object is reproducible.""" + if generator is None: + return None + if isinstance(generator, list): + for gen in generator: + if gen is not None: + gen.manual_seed(int(gen.initial_seed())) + return generator + generator.manual_seed(int(generator.initial_seed())) + return generator + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + loaded = self._read_id2label_from_model_index(getattr(self.config, "_name_or_path", None)) + if loaded: + self._id2label = loaded + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _read_id2label_from_model_index(variant_path: Optional[str]) -> Dict[int, str]: + if not variant_path: + return {} + model_index_path = Path(variant_path).resolve() / "model_index.json" + if not model_index_path.exists(): + return {} + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if not isinstance(id2label, dict): + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + self._ensure_labels_loaded() + if not self.labels: + raise ValueError("No labels loaded. Ensure `id2label` exists in model_index.json.") + labels = [label] if isinstance(label, str) else label + missing = [item for item in labels if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Example valid labels: {preview}, ...") + return [self.labels[item] for item in labels] + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) # type: ignore[arg-type] + return [int(class_id) for class_id in class_labels] + + def _recommended_noise_scale(self) -> float: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in RECOMMENDED_NOISE_BY_MODEL: + return RECOMMENDED_NOISE_BY_MODEL[model_type] + image_size = int(self.transformer.config.sample_size) + return {256: 1.0, 512: 2.0}.get(image_size, 1.0) + + def _default_cfg(self) -> Dict[str, float]: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in DEFAULT_CFG_BY_MODEL: + return dict(DEFAULT_CFG_BY_MODEL[model_type]) + return {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8} + + def predict_u( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + guidance_interval_min: torch.Tensor, + guidance_interval_max: torch.Tensor, + ) -> torch.Tensor: + output = self.transformer( + sample=sample, + timestep=timestep, + class_labels=class_labels, + h=h, + omega=omega, + guidance_interval_min=guidance_interval_min, + guidance_interval_max=guidance_interval_max, + return_dict=True, + ) + return output.u + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + num_inference_steps: int = 1, + guidance_scale: Optional[float] = None, + guidance_interval_min: Optional[float] = None, + guidance_interval_max: Optional[float] = None, + noise_scale: Optional[float] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + if output_type not in {"pil", "np", "pt"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt'.") + + defaults = self._default_cfg() + if guidance_scale is None: + guidance_scale = defaults["guidance_scale"] + if guidance_interval_min is None: + guidance_interval_min = defaults["guidance_interval_min"] + if guidance_interval_max is None: + guidance_interval_max = defaults["guidance_interval_max"] + if noise_scale is None: + noise_scale = self._recommended_noise_scale() + + class_label_ids = self._normalize_class_labels(class_labels) + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + generator = self._prepare_generator(generator) + image_size = int(self.transformer.config.sample_size) + channels = int(self.transformer.config.in_channels) + null_class_val = int( + getattr(self.transformer.config, "num_classes", getattr(self.transformer.config, "num_class_embeds", 1000)) + ) + + latents = randn_tensor( + shape=(batch_size, channels, image_size, image_size), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) * noise_scale + + class_labels_t = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1) + class_labels_t = class_labels_t.clamp(0, null_class_val - 1) + + device = latents.device + dtype = latents.dtype + omega = torch.full((batch_size,), guidance_scale, device=device, dtype=dtype) + t_min = torch.full((batch_size,), guidance_interval_min, device=device, dtype=dtype) + t_max = torch.full((batch_size,), guidance_interval_max, device=device, dtype=dtype) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + timesteps = self.scheduler.timesteps + + for step_index in self.progress_bar(range(num_inference_steps)): + t = timesteps[step_index] + t_next = timesteps[step_index + 1] + h = (t - t_next).expand(batch_size).to(device=device, dtype=dtype) + t_batch = t.expand(batch_size).to(device=device, dtype=dtype) + + u = self.predict_u( + sample=latents, + timestep=t_batch, + class_labels=class_labels_t, + h=h, + omega=omega, + guidance_interval_min=t_min, + guidance_interval_max=t_max, + ) + latents = self.scheduler.step(u, t, latents, **extra_step_kwargs).prev_sample + + images_pt = ((latents.float().clamp(-1, 1) + 1.0) / 2.0).cpu() + if output_type == "pt": + images = images_pt + elif output_type == "np": + images = images_pt.permute(0, 2, 3, 1).numpy() + else: + images = self.numpy_to_pil(images_pt.permute(0, 2, 3, 1).numpy()) + + self.maybe_free_model_hooks() + + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) + + +PMFPipelineOutput = ImagePipelineOutput + +__all__ = ["PMFPipeline", "PMFPipelineOutput"] diff --git a/pMF-H-32-SIM/scheduler/scheduler_config.json b/pMF-H-32-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..341ff049a80407b660e0c785f274b39768b8ebce --- /dev/null +++ b/pMF-H-32-SIM/scheduler/scheduler_config.json @@ -0,0 +1,6 @@ +{ + "_class_name": "PMFScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000, + "stochastic_sampling": false +} diff --git a/pMF-H-32-SIM/scheduler/scheduling_pmf.py b/pMF-H-32-SIM/scheduler/scheduling_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..e4a599c65bb4c64bb5296ab66e5495ac0d936139 --- /dev/null +++ b/pMF-H-32-SIM/scheduler/scheduling_pmf.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class PMFSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class PMFScheduler(SchedulerMixin, ConfigMixin): + """Scheduler for Pixel Mean Flow sampling (t: 1 -> 0).""" + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + stochastic_sampling: bool = False, + ): + del num_train_timesteps + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> None: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.flatten()[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + @staticmethod + def _expand_time(value: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + value = value.to(dtype=sample.dtype, device=sample.device) + while value.ndim < sample.ndim: + value = value.unsqueeze(-1) + return value + + def _sample_noise( + self, + sample: torch.Tensor, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> torch.Tensor: + return randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[PMFSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = self._expand_time(t - t_next, sample) + + if self.config.stochastic_sampling: + t_expanded = self._expand_time(t, sample) + t_next_expanded = self._expand_time(t_next, sample) + x0 = sample - t_expanded * model_output + noise = self._sample_noise(sample, generator=generator) + prev_sample = (1.0 - t_next_expanded) * x0 + t_next_expanded * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return PMFSchedulerOutput(prev_sample=prev_sample) diff --git a/pMF-H-32-SIM/transformer/config.json b/pMF-H-32-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..94daed3b9e5149362f1db9f7832e6174f399e8c1 --- /dev/null +++ b/pMF-H-32-SIM/transformer/config.json @@ -0,0 +1,26 @@ +{ + "_class_name": "PMFTransformer2DModel", + "_diffusers_version": "0.38.0", + "aux_head_depth": 8, + "bottleneck_dim": 256, + "depth": 48, + "embedding_init_constant": 1.0, + "eval_mode": true, + "hidden_size": 1280, + "in_channels": 3, + "mlp_ratio": 2.6666666666666665, + "model_type": "pMF-H/32", + "norm_eps": 1e-06, + "num_attention_heads": 16, + "num_cfg_tokens": 4, + "num_class_embeds": null, + "num_class_tokens": 8, + "num_classes": 1000, + "num_interval_tokens": 2, + "num_time_tokens": 4, + "patch_size": 32, + "sample_size": 512, + "t_clip_min": 0.05, + "token_init_constant": 1.0, + "weight_init_constant": 0.32 +} diff --git a/pMF-H-32-SIM/transformer/diffusion_pytorch_model.safetensors b/pMF-H-32-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a848d7d6e6832fcfc7423458c72b01ae2d915a97 --- /dev/null +++ b/pMF-H-32-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd6ad5fdff4ced901fb13f77c9d90bba8b1f781d363093956ec319f679b8149d +size 3836269544 diff --git a/pMF-H-32-SIM/transformer/transformer_pmf.py b/pMF-H-32-SIM/transformer/transformer_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f97067569ec0a4d038069edca73b9c1d003596 --- /dev/null +++ b/pMF-H-32-SIM/transformer/transformer_pmf.py @@ -0,0 +1,665 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from math import sqrt +from typing import Dict, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm +from diffusers.utils import BaseOutput + + +PMF_PRESET_CONFIGS: Dict[str, Dict[str, object]] = { + "pMF-B/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-B/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-H/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, + "pMF-H/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + +# Legacy torch repo keys (pmfDiT_*) +LEGACY_MODEL_ALIASES: Dict[str, str] = { + "pmfDiT_B_16": "pMF-B/16", + "pmfDiT_B_32": "pMF-B/32", + "pmfDiT_L_16": "pMF-L/16", + "pmfDiT_L_32": "pMF-L/32", + "pmfDiT_H_16": "pMF-H/16", + "pmfDiT_H_32": "pMF-H/32", +} + + +@dataclass +class PMFTransformer2DOutput(BaseOutput): + u: torch.Tensor + v: Optional[torch.Tensor] = None + + +def remap_legacy_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Map wrapper/backbone keys from legacy checkpoints to native PMFTransformer2DModel keys.""" + remapped: Dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + for prefix in ("transformer.", "net."): + if new_key.startswith(prefix): + new_key = new_key[len(prefix) :] + break + # Official PyTorch checkpoints use TorchLinear/TorchEmbedding wrappers. + new_key = new_key.replace("._flax_linear", "").replace("._flax_embedding", "") + if new_key == "rope_freqs": + continue + remapped[new_key] = value + return remapped + + +def config_from_legacy(config: Dict[str, object]) -> Dict[str, object]: + """Build native config kwargs from a legacy config.json dict.""" + model_type = config.get("model_type") or config.get("model_name") or config.get("model_str") + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type not in PMF_PRESET_CONFIGS: + raise ValueError(f"Unknown pMF preset '{model_type}'. Known: {list(PMF_PRESET_CONFIGS)}") + + preset = dict(PMF_PRESET_CONFIGS[model_type]) + preset["num_classes"] = int(config.get("num_class_embeds") or config.get("num_classes") or 1000) + preset["model_type"] = model_type + if config.get("sample_size") is not None: + preset["sample_size"] = int(config["sample_size"]) + if config.get("eval_mode") is not None: + preset["eval_mode"] = bool(config["eval_mode"]) + return preset + + +def _scaled_linear( + in_features: int, + out_features: int, + *, + bias: bool = True, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + bias_init: str = "zeros", +) -> nn.Linear: + layer = nn.Linear(in_features, out_features, bias=bias) + if weight_init == "scaled_variance": + std = init_constant / sqrt(in_features) + nn.init.normal_(layer.weight, std=std) + elif weight_init == "zeros": + nn.init.zeros_(layer.weight) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + + if bias: + if bias_init == "zeros": + nn.init.zeros_(layer.bias) + else: + raise ValueError(f"Invalid bias_init: {bias_init}") + return layer + + +class PMFTimestepEmbedder(nn.Module): + def __init__( + self, + hidden_size: int, + frequency_embedding_size: int = 256, + init_constant: float = 1.0, + ): + super().__init__() + init_kwargs = dict( + out_features=hidden_size, + bias=True, + weight_init="scaled_variance", + init_constant=init_constant, + bias_init="zeros", + ) + self.mlp = nn.Sequential( + _scaled_linear(frequency_embedding_size, **init_kwargs), + nn.SiLU(), + _scaled_linear(hidden_size, **init_kwargs), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + half = dim // 2 + compute_dtype = t.dtype + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=compute_dtype, device=t.device) + / half + ) + args = t[:, None] * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + return self.mlp(t_freq) + + +class PMFLabelEmbedder(nn.Module): + def __init__(self, num_classes: int, hidden_size: int, init_constant: float = 1.0): + super().__init__() + self.embedding_table = nn.Embedding(num_classes + 1, hidden_size) + nn.init.normal_(self.embedding_table.weight, std=init_constant / sqrt(hidden_size)) + + def forward(self, labels: torch.Tensor) -> torch.Tensor: + return self.embedding_table(labels) + + +class PMFBottleneckPatchEmbedder(nn.Module): + def __init__( + self, + input_size: int, + patch_size: int, + pca_channels: int, + in_channels: int, + hidden_size: int, + bias: bool = True, + ): + super().__init__() + self.patch_size = (patch_size, patch_size) + self.num_patches = (input_size // patch_size) ** 2 + self.proj1 = nn.Conv2d( + in_channels, + pca_channels, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + ) + self.proj2 = nn.Conv2d(pca_channels, hidden_size, kernel_size=1, stride=1, bias=bias) + + kh = kw = patch_size + fan_in = kh * kw * in_channels + fan_out = pca_channels + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj1.weight, -limit, limit) + fan_in = pca_channels + fan_out = hidden_size + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj2.weight, -limit, limit) + if bias: + nn.init.zeros_(self.proj1.bias) + nn.init.zeros_(self.proj2.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj2(self.proj1(x)) + return x.flatten(2).transpose(1, 2) + + +def precompute_rope_freqs(dim: int, seq_len: int, theta: float = 10000.0) -> torch.Tensor: + dim = dim // 2 + grid_size = int(seq_len**0.5) + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + positions = torch.arange(grid_size, dtype=torch.float32) + freqs_h = torch.einsum("i,j->ij", positions, freqs) + freqs_w = torch.einsum("i,j->ij", positions, freqs) + freqs_2d = torch.cat( + [ + torch.tile(freqs_h[:, None, :], (1, grid_size, 1)), + torch.tile(freqs_w[None, :, :], (grid_size, 1, 1)), + ], + dim=-1, + ) + real = torch.cos(freqs_2d).reshape(seq_len, dim) + imag = torch.sin(freqs_2d).reshape(seq_len, dim) + return torch.complex(real, imag) + + +def apply_rotary_pos_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + x_float = x.to(torch.float32) + x_complex = torch.view_as_complex(x_float.reshape(*x_float.shape[:-1], -1, 2).contiguous()) + freqs_cis = freqs_cis.unsqueeze(0).unsqueeze(2) + token_count = freqs_cis.shape[1] + x_rotated = x_complex.clone() + x_rotated[:, -token_count:, :] = x_complex[:, -token_count:, :] * freqs_cis + x_out = torch.view_as_real(x_rotated).flatten(-2) + return x_out.to(x.dtype) + + +class PMFAttention(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + init_kwargs = dict( + bias=False, + weight_init="scaled_variance", + init_constant=weight_init_constant, + ) + self.q_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.k_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.v_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.out_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.q_norm = RMSNorm(self.head_dim, eps=eps) + self.k_norm = RMSNorm(self.head_dim, eps=eps) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + batch_size, seq_len, channels = x.shape + q = self.q_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + k = self.k_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + v = self.v_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + + q = self.q_norm(q) + k = self.k_norm(k) + q = apply_rotary_pos_emb(q, rope_freqs) + k = apply_rotary_pos_emb(k, rope_freqs) + + query = q / math.sqrt(self.head_dim) + attn_weights = torch.einsum("bqhd,bkhd->bhqk", query, k) + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(v.dtype) + attn = torch.einsum("bhqk,bkhd->bqhd", attn_weights, v) + attn = attn.reshape(batch_size, seq_len, channels) + return self.out_proj(attn) + + +class PMFSwiGLUMlp(nn.Module): + def __init__(self, dim: int, hidden_dim: int, weight_init_constant: float = 0.32): + super().__init__() + init_kwargs = dict(bias=False, weight_init="scaled_variance", init_constant=weight_init_constant) + self.w1 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w3 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w2 = _scaled_linear(hidden_dim, dim, **init_kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + + +class PMFTransformerBlock(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float = 8 / 3, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.norm1 = RMSNorm(hidden_size, eps=eps) + self.attn = PMFAttention(hidden_size, num_heads, weight_init_constant=weight_init_constant, eps=eps) + self.norm2 = RMSNorm(hidden_size, eps=eps) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + if hidden_size > 1024: + mlp_hidden_dim = (mlp_hidden_dim + 7) // 8 * 8 + self.mlp = PMFSwiGLUMlp(hidden_size, mlp_hidden_dim, weight_init_constant=weight_init_constant) + self.attn_scale = nn.Parameter(torch.zeros(hidden_size)) + self.mlp_scale = nn.Parameter(torch.zeros(hidden_size)) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), rope_freqs) * self.attn_scale + x = x + self.mlp(self.norm2(x)) * self.mlp_scale + return x + + +class PMFFinalLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int, eps: float = 1e-6): + super().__init__() + self.norm = RMSNorm(hidden_size, eps=eps) + self.linear = _scaled_linear( + hidden_size, + patch_size * patch_size * out_channels, + bias=True, + weight_init="zeros", + bias_init="zeros", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(self.norm(x)) + + +class PMFTransformer2DModel(ModelMixin, ConfigMixin): + """Native diffusers implementation of the pMF DiT backbone.""" + + _supports_gradient_checkpointing = True + _skip_layerwise_casting_patterns = ["pos_embed", "rope_freqs"] + + @register_to_config + def __init__( + self, + sample_size: int = 256, + patch_size: int = 16, + in_channels: int = 3, + hidden_size: int = 768, + depth: int = 16, + num_attention_heads: int = 12, + mlp_ratio: float = 8 / 3, + num_classes: int = 1000, + bottleneck_dim: int = 128, + aux_head_depth: int = 8, + num_class_tokens: int = 8, + num_time_tokens: int = 4, + num_cfg_tokens: int = 4, + num_interval_tokens: int = 2, + token_init_constant: float = 1.0, + embedding_init_constant: float = 1.0, + weight_init_constant: float = 0.32, + eval_mode: bool = True, + model_type: str | None = None, + num_class_embeds: int | None = None, + t_clip_min: float = 0.05, + norm_eps: float = 1e-6, + ): + super().__init__() + if num_class_embeds is not None: + num_classes = int(num_class_embeds) + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type in PMF_PRESET_CONFIGS: + preset = PMF_PRESET_CONFIGS[model_type] + sample_size = int(preset["sample_size"]) + patch_size = int(preset["patch_size"]) + hidden_size = int(preset["hidden_size"]) + depth = int(preset["depth"]) + num_attention_heads = int(preset["num_attention_heads"]) + bottleneck_dim = int(preset["bottleneck_dim"]) + aux_head_depth = int(preset["aux_head_depth"]) + + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.depth = depth + self.num_attention_heads = num_attention_heads + self.aux_head_depth = aux_head_depth + self.num_class_tokens = num_class_tokens + self.num_time_tokens = num_time_tokens + self.num_cfg_tokens = num_cfg_tokens + self.num_interval_tokens = num_interval_tokens + self.prefix_tokens = ( + num_class_tokens + num_cfg_tokens + 2 * num_interval_tokens + num_time_tokens + ) + self.t_clip_min = t_clip_min + self.eval_mode = eval_mode + self.gradient_checkpointing = False + + self.x_embedder = PMFBottleneckPatchEmbedder( + sample_size, + patch_size, + bottleneck_dim, + in_channels, + hidden_size, + bias=True, + ) + embed_kwargs = dict(hidden_size=hidden_size, init_constant=embedding_init_constant) + self.h_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.omega_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_start_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_end_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.y_embedder = PMFLabelEmbedder(num_classes, hidden_size, init_constant=embedding_init_constant) + + token_std = token_init_constant / math.sqrt(hidden_size) + self.time_tokens = nn.Parameter(torch.randn(1, num_time_tokens, hidden_size) * token_std) + self.class_tokens = nn.Parameter(torch.randn(1, num_class_tokens, hidden_size) * token_std) + self.omega_tokens = nn.Parameter(torch.randn(1, num_cfg_tokens, hidden_size) * token_std) + self.t_min_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + self.t_max_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + + total_tokens = self.x_embedder.num_patches + self.prefix_tokens + self.pos_embed = nn.Parameter(torch.randn(1, total_tokens, hidden_size) * 0.02) + + head_dim = hidden_size // num_attention_heads + self.register_buffer( + "rope_freqs", + precompute_rope_freqs(head_dim, self.x_embedder.num_patches), + persistent=False, + ) + + shared_depth = depth - aux_head_depth + block_kwargs = dict( + hidden_size=hidden_size, + num_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + weight_init_constant=weight_init_constant, + eps=norm_eps, + ) + self.shared_blocks = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(shared_depth)]) + self.u_heads = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth)]) + self.v_heads = nn.ModuleList( + [PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth if not eval_mode else 0)] + ) + self.u_final_layer = PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + self.v_final_layer = ( + PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + if not eval_mode + else None + ) + + def _build_sequence( + self, + sample: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + t_min: torch.Tensor, + t_max: torch.Tensor, + class_labels: torch.Tensor, + ) -> torch.Tensor: + x_embed = self.x_embedder(sample) + h_embed = self.h_embedder(h) + omega_embed = self.omega_embedder(1 - 1 / omega) + t_min_embed = self.cfg_t_start_embedder(t_min) + t_max_embed = self.cfg_t_end_embedder(t_max) + y_embed = self.y_embedder(class_labels) + + time_tokens = self.time_tokens + h_embed.unsqueeze(1) + omega_tokens = self.omega_tokens + omega_embed.unsqueeze(1) + t_min_tokens = self.t_min_tokens + t_min_embed.unsqueeze(1) + t_max_tokens = self.t_max_tokens + t_max_embed.unsqueeze(1) + class_tokens = self.class_tokens + y_embed.unsqueeze(1) + + seq = torch.cat( + [class_tokens, omega_tokens, t_min_tokens, t_max_tokens, time_tokens, x_embed], + dim=1, + ) + return seq + self.pos_embed + + def _unpatchify(self, tokens: torch.Tensor) -> torch.Tensor: + batch_size = tokens.shape[0] + patch = self.patch_size + grid = int(tokens.shape[1] ** 0.5) + channels = self.out_channels + x = tokens.reshape(batch_size, grid, grid, patch, patch, channels) + x = torch.einsum("nhwpqc->nchpwq", x) + return x.reshape(batch_size, channels, grid * patch, grid * patch) + + def forward( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: Optional[torch.Tensor] = None, + omega: Optional[torch.Tensor] = None, + guidance_interval_min: Optional[torch.Tensor] = None, + guidance_interval_max: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> PMFTransformer2DOutput | Tuple[torch.Tensor, Optional[torch.Tensor]]: + batch_size = sample.shape[0] + timestep = self._expand_batch(timestep, batch_size, sample.device, sample.dtype) + h = self._expand_batch(h if h is not None else timestep, batch_size, sample.device, sample.dtype) + omega = self._expand_batch( + omega if omega is not None else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_min = self._expand_batch( + guidance_interval_min + if guidance_interval_min is not None + else torch.zeros(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_max = self._expand_batch( + guidance_interval_max + if guidance_interval_max is not None + else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + + seq = self._build_sequence(sample, h, omega, guidance_interval_min, guidance_interval_max, class_labels) + rope_freqs = self.rope_freqs.to(device=sample.device) + + for block in self.shared_blocks: + if self.training and self.gradient_checkpointing: + seq = torch.utils.checkpoint.checkpoint(block, seq, rope_freqs, use_reentrant=False) + else: + seq = block(seq, rope_freqs) + + u_seq = v_seq = seq + for block in self.u_heads: + if self.training and self.gradient_checkpointing: + u_seq = torch.utils.checkpoint.checkpoint(block, u_seq, rope_freqs, use_reentrant=False) + else: + u_seq = block(u_seq, rope_freqs) + + for block in self.v_heads: + if self.training and self.gradient_checkpointing: + v_seq = torch.utils.checkpoint.checkpoint(block, v_seq, rope_freqs, use_reentrant=False) + else: + v_seq = block(v_seq, rope_freqs) + + u_tokens = u_seq[:, self.prefix_tokens :] + u_pred = self._unpatchify(self.u_final_layer(u_tokens)) + t = timestep.reshape(batch_size, 1, 1, 1) + u = (sample - u_pred) / torch.clamp(t, min=self.t_clip_min) + + v = None + if self.v_final_layer is not None: + v_tokens = v_seq[:, self.prefix_tokens :] + v_pred = self._unpatchify(self.v_final_layer(v_tokens)) + v = (sample - v_pred) / torch.clamp(t, min=self.t_clip_min) + + if not return_dict: + return (u, v) + return PMFTransformer2DOutput(u=u, v=v) + + @staticmethod + def _expand_batch( + value: torch.Tensor, + batch_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + value = torch.as_tensor(value, device=device, dtype=dtype) + if value.ndim == 0: + value = value.reshape(1) + if value.shape[0] == 1 and batch_size > 1: + value = value.expand(batch_size) + return value.reshape(batch_size) + + @classmethod + def from_pmf_checkpoint( + cls, + checkpoint_path: str, + model_type: str | None = None, + map_location: str = "cpu", + strict: bool = False, + ) -> Tuple["PMFTransformer2DModel", Dict[str, object]]: + checkpoint = torch.load(checkpoint_path, map_location=map_location, weights_only=False) + if isinstance(checkpoint, dict) and "state_dict" in checkpoint: + state_dict = checkpoint["state_dict"] + else: + state_dict = checkpoint + + if model_type is None: + for key in ("model_type", "model_str", "model"): + if isinstance(checkpoint, dict) and key in checkpoint: + model_type = checkpoint[key] + break + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type is None: + raise ValueError("model_type is required when it cannot be inferred from the checkpoint.") + + config = dict(PMF_PRESET_CONFIGS[model_type]) + config["model_type"] = model_type + config["eval_mode"] = True + model = cls(**config) + model.load_state_dict(remap_legacy_state_dict(state_dict), strict=strict) + metadata = {"checkpoint_path": checkpoint_path, "model_type": model_type} + return model, metadata + + def to_pmf_checkpoint(self, prefix: str = "net.") -> Dict[str, torch.Tensor]: + state_dict: Dict[str, torch.Tensor] = {} + for key, value in self.state_dict().items(): + state_dict[f"{prefix}{key}"] = value.detach().cpu() + return state_dict + + @property + def net(self): + return self + + +PMFDiffusersModel = PMFTransformer2DModel diff --git a/pMF-L-16-SIM/README.md b/pMF-L-16-SIM/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6ef0425fbbfe7511f83ede004f33ed01fc662b22 --- /dev/null +++ b/pMF-L-16-SIM/README.md @@ -0,0 +1,44 @@ +--- +license: mit +library_name: diffusers +pipeline_tag: text-to-image +tags: +- diffusers +- pmf +- image-generation +- class-conditional +- imagenet +inference: true +--- + +# pMF-L-16-SIM + +Self-contained Diffusers variant for **pMF-L/16 (FD-SIM post-trained)** (Pixel Mean Flows). + +Recommended settings: `guidance_scale=7.0`, interval `[0.2, 0.7]`, `noise_scale=1.0`. + +## Load + +```python +from pathlib import Path +from diffusers import DiffusionPipeline +import torch + +model_dir = Path("./pMF-L-16-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + local_files_only=True, + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.bfloat16, +).to("cuda") + +image = pipe( + class_labels=207, + num_inference_steps=1, + guidance_scale=7.0, + guidance_interval_min=0.2, + guidance_interval_max=0.7, + noise_scale=1.0, +).images[0] +``` diff --git a/pMF-L-16-SIM/model_index.json b/pMF-L-16-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..ec00b5bd38c72b4310e6e3cb2c65ce31ef2427b4 --- /dev/null +++ b/pMF-L-16-SIM/model_index.json @@ -0,0 +1,1017 @@ +{ + "_class_name": [ + "pipeline", + "PMFPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_pmf", + "PMFScheduler" + ], + "transformer": [ + "transformer_pmf", + "PMFTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + } +} diff --git a/pMF-L-16-SIM/pipeline.py b/pMF-L-16-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..314f675b581fe55ede8cbc0d1f2e1f850e5e9f34 --- /dev/null +++ b/pMF-L-16-SIM/pipeline.py @@ -0,0 +1,289 @@ +"""Hub custom pipeline: PMFPipeline.""" + +from __future__ import annotations + +import inspect +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +DEFAULT_CFG_BY_MODEL: Dict[str, Dict[str, float]] = { + "pMF-B/16": {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8}, + "pMF-B/32": {"guidance_scale": 6.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.7}, + "pMF-L/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.7}, + "pMF-L/32": {"guidance_scale": 7.5, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/32": {"guidance_scale": 5.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.6}, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + + +class PMFPipeline(DiffusionPipeline): + model_cpu_offload_seq = "transformer" + + def __init__( + self, + transformer: Any, + scheduler: Any, + id2label: Optional[Dict[Union[int, str], str]] = None, + ) -> None: + super().__init__() + if scheduler is None: + raise ValueError("PMFPipeline requires a scheduler loaded from the checkpoint.") + self.register_modules(transformer=transformer, scheduler=scheduler) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + + @staticmethod + def prepare_extra_step_kwargs( + scheduler: Any, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + eta: float | None = None, + ) -> Dict[str, Any]: + kwargs: Dict[str, Any] = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + @staticmethod + def _prepare_generator( + generator: Optional[Union[torch.Generator, List[torch.Generator]]], + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + """Reset generator state so repeated `__call__` with the same object is reproducible.""" + if generator is None: + return None + if isinstance(generator, list): + for gen in generator: + if gen is not None: + gen.manual_seed(int(gen.initial_seed())) + return generator + generator.manual_seed(int(generator.initial_seed())) + return generator + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + loaded = self._read_id2label_from_model_index(getattr(self.config, "_name_or_path", None)) + if loaded: + self._id2label = loaded + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _read_id2label_from_model_index(variant_path: Optional[str]) -> Dict[int, str]: + if not variant_path: + return {} + model_index_path = Path(variant_path).resolve() / "model_index.json" + if not model_index_path.exists(): + return {} + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if not isinstance(id2label, dict): + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + self._ensure_labels_loaded() + if not self.labels: + raise ValueError("No labels loaded. Ensure `id2label` exists in model_index.json.") + labels = [label] if isinstance(label, str) else label + missing = [item for item in labels if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Example valid labels: {preview}, ...") + return [self.labels[item] for item in labels] + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) # type: ignore[arg-type] + return [int(class_id) for class_id in class_labels] + + def _recommended_noise_scale(self) -> float: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in RECOMMENDED_NOISE_BY_MODEL: + return RECOMMENDED_NOISE_BY_MODEL[model_type] + image_size = int(self.transformer.config.sample_size) + return {256: 1.0, 512: 2.0}.get(image_size, 1.0) + + def _default_cfg(self) -> Dict[str, float]: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in DEFAULT_CFG_BY_MODEL: + return dict(DEFAULT_CFG_BY_MODEL[model_type]) + return {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8} + + def predict_u( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + guidance_interval_min: torch.Tensor, + guidance_interval_max: torch.Tensor, + ) -> torch.Tensor: + output = self.transformer( + sample=sample, + timestep=timestep, + class_labels=class_labels, + h=h, + omega=omega, + guidance_interval_min=guidance_interval_min, + guidance_interval_max=guidance_interval_max, + return_dict=True, + ) + return output.u + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + num_inference_steps: int = 1, + guidance_scale: Optional[float] = None, + guidance_interval_min: Optional[float] = None, + guidance_interval_max: Optional[float] = None, + noise_scale: Optional[float] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + if output_type not in {"pil", "np", "pt"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt'.") + + defaults = self._default_cfg() + if guidance_scale is None: + guidance_scale = defaults["guidance_scale"] + if guidance_interval_min is None: + guidance_interval_min = defaults["guidance_interval_min"] + if guidance_interval_max is None: + guidance_interval_max = defaults["guidance_interval_max"] + if noise_scale is None: + noise_scale = self._recommended_noise_scale() + + class_label_ids = self._normalize_class_labels(class_labels) + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + generator = self._prepare_generator(generator) + image_size = int(self.transformer.config.sample_size) + channels = int(self.transformer.config.in_channels) + null_class_val = int( + getattr(self.transformer.config, "num_classes", getattr(self.transformer.config, "num_class_embeds", 1000)) + ) + + latents = randn_tensor( + shape=(batch_size, channels, image_size, image_size), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) * noise_scale + + class_labels_t = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1) + class_labels_t = class_labels_t.clamp(0, null_class_val - 1) + + device = latents.device + dtype = latents.dtype + omega = torch.full((batch_size,), guidance_scale, device=device, dtype=dtype) + t_min = torch.full((batch_size,), guidance_interval_min, device=device, dtype=dtype) + t_max = torch.full((batch_size,), guidance_interval_max, device=device, dtype=dtype) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + timesteps = self.scheduler.timesteps + + for step_index in self.progress_bar(range(num_inference_steps)): + t = timesteps[step_index] + t_next = timesteps[step_index + 1] + h = (t - t_next).expand(batch_size).to(device=device, dtype=dtype) + t_batch = t.expand(batch_size).to(device=device, dtype=dtype) + + u = self.predict_u( + sample=latents, + timestep=t_batch, + class_labels=class_labels_t, + h=h, + omega=omega, + guidance_interval_min=t_min, + guidance_interval_max=t_max, + ) + latents = self.scheduler.step(u, t, latents, **extra_step_kwargs).prev_sample + + images_pt = ((latents.float().clamp(-1, 1) + 1.0) / 2.0).cpu() + if output_type == "pt": + images = images_pt + elif output_type == "np": + images = images_pt.permute(0, 2, 3, 1).numpy() + else: + images = self.numpy_to_pil(images_pt.permute(0, 2, 3, 1).numpy()) + + self.maybe_free_model_hooks() + + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) + + +PMFPipelineOutput = ImagePipelineOutput + +__all__ = ["PMFPipeline", "PMFPipelineOutput"] diff --git a/pMF-L-16-SIM/scheduler/scheduler_config.json b/pMF-L-16-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..341ff049a80407b660e0c785f274b39768b8ebce --- /dev/null +++ b/pMF-L-16-SIM/scheduler/scheduler_config.json @@ -0,0 +1,6 @@ +{ + "_class_name": "PMFScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000, + "stochastic_sampling": false +} diff --git a/pMF-L-16-SIM/scheduler/scheduling_pmf.py b/pMF-L-16-SIM/scheduler/scheduling_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..e4a599c65bb4c64bb5296ab66e5495ac0d936139 --- /dev/null +++ b/pMF-L-16-SIM/scheduler/scheduling_pmf.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class PMFSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class PMFScheduler(SchedulerMixin, ConfigMixin): + """Scheduler for Pixel Mean Flow sampling (t: 1 -> 0).""" + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + stochastic_sampling: bool = False, + ): + del num_train_timesteps + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> None: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.flatten()[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + @staticmethod + def _expand_time(value: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + value = value.to(dtype=sample.dtype, device=sample.device) + while value.ndim < sample.ndim: + value = value.unsqueeze(-1) + return value + + def _sample_noise( + self, + sample: torch.Tensor, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> torch.Tensor: + return randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[PMFSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = self._expand_time(t - t_next, sample) + + if self.config.stochastic_sampling: + t_expanded = self._expand_time(t, sample) + t_next_expanded = self._expand_time(t_next, sample) + x0 = sample - t_expanded * model_output + noise = self._sample_noise(sample, generator=generator) + prev_sample = (1.0 - t_next_expanded) * x0 + t_next_expanded * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return PMFSchedulerOutput(prev_sample=prev_sample) diff --git a/pMF-L-16-SIM/transformer/config.json b/pMF-L-16-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8fc46c306449dbee27c5dd63713572d898176476 --- /dev/null +++ b/pMF-L-16-SIM/transformer/config.json @@ -0,0 +1,26 @@ +{ + "_class_name": "PMFTransformer2DModel", + "_diffusers_version": "0.38.0", + "aux_head_depth": 8, + "bottleneck_dim": 128, + "depth": 32, + "embedding_init_constant": 1.0, + "eval_mode": true, + "hidden_size": 1024, + "in_channels": 3, + "mlp_ratio": 2.6666666666666665, + "model_type": "pMF-L/16", + "norm_eps": 1e-06, + "num_attention_heads": 16, + "num_cfg_tokens": 4, + "num_class_embeds": null, + "num_class_tokens": 8, + "num_classes": 1000, + "num_interval_tokens": 2, + "num_time_tokens": 4, + "patch_size": 16, + "sample_size": 256, + "t_clip_min": 0.05, + "token_init_constant": 1.0, + "weight_init_constant": 0.32 +} diff --git a/pMF-L-16-SIM/transformer/diffusion_pytorch_model.safetensors b/pMF-L-16-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b147df25710fc10330394fb2455dfc9046213cd1 --- /dev/null +++ b/pMF-L-16-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2afad60af499bf3b20131836c583c98691d88b78e6a05e94d299a112dc8619fa +size 1641329160 diff --git a/pMF-L-16-SIM/transformer/transformer_pmf.py b/pMF-L-16-SIM/transformer/transformer_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f97067569ec0a4d038069edca73b9c1d003596 --- /dev/null +++ b/pMF-L-16-SIM/transformer/transformer_pmf.py @@ -0,0 +1,665 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from math import sqrt +from typing import Dict, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm +from diffusers.utils import BaseOutput + + +PMF_PRESET_CONFIGS: Dict[str, Dict[str, object]] = { + "pMF-B/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-B/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-H/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, + "pMF-H/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + +# Legacy torch repo keys (pmfDiT_*) +LEGACY_MODEL_ALIASES: Dict[str, str] = { + "pmfDiT_B_16": "pMF-B/16", + "pmfDiT_B_32": "pMF-B/32", + "pmfDiT_L_16": "pMF-L/16", + "pmfDiT_L_32": "pMF-L/32", + "pmfDiT_H_16": "pMF-H/16", + "pmfDiT_H_32": "pMF-H/32", +} + + +@dataclass +class PMFTransformer2DOutput(BaseOutput): + u: torch.Tensor + v: Optional[torch.Tensor] = None + + +def remap_legacy_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Map wrapper/backbone keys from legacy checkpoints to native PMFTransformer2DModel keys.""" + remapped: Dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + for prefix in ("transformer.", "net."): + if new_key.startswith(prefix): + new_key = new_key[len(prefix) :] + break + # Official PyTorch checkpoints use TorchLinear/TorchEmbedding wrappers. + new_key = new_key.replace("._flax_linear", "").replace("._flax_embedding", "") + if new_key == "rope_freqs": + continue + remapped[new_key] = value + return remapped + + +def config_from_legacy(config: Dict[str, object]) -> Dict[str, object]: + """Build native config kwargs from a legacy config.json dict.""" + model_type = config.get("model_type") or config.get("model_name") or config.get("model_str") + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type not in PMF_PRESET_CONFIGS: + raise ValueError(f"Unknown pMF preset '{model_type}'. Known: {list(PMF_PRESET_CONFIGS)}") + + preset = dict(PMF_PRESET_CONFIGS[model_type]) + preset["num_classes"] = int(config.get("num_class_embeds") or config.get("num_classes") or 1000) + preset["model_type"] = model_type + if config.get("sample_size") is not None: + preset["sample_size"] = int(config["sample_size"]) + if config.get("eval_mode") is not None: + preset["eval_mode"] = bool(config["eval_mode"]) + return preset + + +def _scaled_linear( + in_features: int, + out_features: int, + *, + bias: bool = True, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + bias_init: str = "zeros", +) -> nn.Linear: + layer = nn.Linear(in_features, out_features, bias=bias) + if weight_init == "scaled_variance": + std = init_constant / sqrt(in_features) + nn.init.normal_(layer.weight, std=std) + elif weight_init == "zeros": + nn.init.zeros_(layer.weight) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + + if bias: + if bias_init == "zeros": + nn.init.zeros_(layer.bias) + else: + raise ValueError(f"Invalid bias_init: {bias_init}") + return layer + + +class PMFTimestepEmbedder(nn.Module): + def __init__( + self, + hidden_size: int, + frequency_embedding_size: int = 256, + init_constant: float = 1.0, + ): + super().__init__() + init_kwargs = dict( + out_features=hidden_size, + bias=True, + weight_init="scaled_variance", + init_constant=init_constant, + bias_init="zeros", + ) + self.mlp = nn.Sequential( + _scaled_linear(frequency_embedding_size, **init_kwargs), + nn.SiLU(), + _scaled_linear(hidden_size, **init_kwargs), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + half = dim // 2 + compute_dtype = t.dtype + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=compute_dtype, device=t.device) + / half + ) + args = t[:, None] * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + return self.mlp(t_freq) + + +class PMFLabelEmbedder(nn.Module): + def __init__(self, num_classes: int, hidden_size: int, init_constant: float = 1.0): + super().__init__() + self.embedding_table = nn.Embedding(num_classes + 1, hidden_size) + nn.init.normal_(self.embedding_table.weight, std=init_constant / sqrt(hidden_size)) + + def forward(self, labels: torch.Tensor) -> torch.Tensor: + return self.embedding_table(labels) + + +class PMFBottleneckPatchEmbedder(nn.Module): + def __init__( + self, + input_size: int, + patch_size: int, + pca_channels: int, + in_channels: int, + hidden_size: int, + bias: bool = True, + ): + super().__init__() + self.patch_size = (patch_size, patch_size) + self.num_patches = (input_size // patch_size) ** 2 + self.proj1 = nn.Conv2d( + in_channels, + pca_channels, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + ) + self.proj2 = nn.Conv2d(pca_channels, hidden_size, kernel_size=1, stride=1, bias=bias) + + kh = kw = patch_size + fan_in = kh * kw * in_channels + fan_out = pca_channels + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj1.weight, -limit, limit) + fan_in = pca_channels + fan_out = hidden_size + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj2.weight, -limit, limit) + if bias: + nn.init.zeros_(self.proj1.bias) + nn.init.zeros_(self.proj2.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj2(self.proj1(x)) + return x.flatten(2).transpose(1, 2) + + +def precompute_rope_freqs(dim: int, seq_len: int, theta: float = 10000.0) -> torch.Tensor: + dim = dim // 2 + grid_size = int(seq_len**0.5) + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + positions = torch.arange(grid_size, dtype=torch.float32) + freqs_h = torch.einsum("i,j->ij", positions, freqs) + freqs_w = torch.einsum("i,j->ij", positions, freqs) + freqs_2d = torch.cat( + [ + torch.tile(freqs_h[:, None, :], (1, grid_size, 1)), + torch.tile(freqs_w[None, :, :], (grid_size, 1, 1)), + ], + dim=-1, + ) + real = torch.cos(freqs_2d).reshape(seq_len, dim) + imag = torch.sin(freqs_2d).reshape(seq_len, dim) + return torch.complex(real, imag) + + +def apply_rotary_pos_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + x_float = x.to(torch.float32) + x_complex = torch.view_as_complex(x_float.reshape(*x_float.shape[:-1], -1, 2).contiguous()) + freqs_cis = freqs_cis.unsqueeze(0).unsqueeze(2) + token_count = freqs_cis.shape[1] + x_rotated = x_complex.clone() + x_rotated[:, -token_count:, :] = x_complex[:, -token_count:, :] * freqs_cis + x_out = torch.view_as_real(x_rotated).flatten(-2) + return x_out.to(x.dtype) + + +class PMFAttention(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + init_kwargs = dict( + bias=False, + weight_init="scaled_variance", + init_constant=weight_init_constant, + ) + self.q_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.k_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.v_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.out_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.q_norm = RMSNorm(self.head_dim, eps=eps) + self.k_norm = RMSNorm(self.head_dim, eps=eps) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + batch_size, seq_len, channels = x.shape + q = self.q_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + k = self.k_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + v = self.v_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + + q = self.q_norm(q) + k = self.k_norm(k) + q = apply_rotary_pos_emb(q, rope_freqs) + k = apply_rotary_pos_emb(k, rope_freqs) + + query = q / math.sqrt(self.head_dim) + attn_weights = torch.einsum("bqhd,bkhd->bhqk", query, k) + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(v.dtype) + attn = torch.einsum("bhqk,bkhd->bqhd", attn_weights, v) + attn = attn.reshape(batch_size, seq_len, channels) + return self.out_proj(attn) + + +class PMFSwiGLUMlp(nn.Module): + def __init__(self, dim: int, hidden_dim: int, weight_init_constant: float = 0.32): + super().__init__() + init_kwargs = dict(bias=False, weight_init="scaled_variance", init_constant=weight_init_constant) + self.w1 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w3 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w2 = _scaled_linear(hidden_dim, dim, **init_kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + + +class PMFTransformerBlock(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float = 8 / 3, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.norm1 = RMSNorm(hidden_size, eps=eps) + self.attn = PMFAttention(hidden_size, num_heads, weight_init_constant=weight_init_constant, eps=eps) + self.norm2 = RMSNorm(hidden_size, eps=eps) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + if hidden_size > 1024: + mlp_hidden_dim = (mlp_hidden_dim + 7) // 8 * 8 + self.mlp = PMFSwiGLUMlp(hidden_size, mlp_hidden_dim, weight_init_constant=weight_init_constant) + self.attn_scale = nn.Parameter(torch.zeros(hidden_size)) + self.mlp_scale = nn.Parameter(torch.zeros(hidden_size)) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), rope_freqs) * self.attn_scale + x = x + self.mlp(self.norm2(x)) * self.mlp_scale + return x + + +class PMFFinalLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int, eps: float = 1e-6): + super().__init__() + self.norm = RMSNorm(hidden_size, eps=eps) + self.linear = _scaled_linear( + hidden_size, + patch_size * patch_size * out_channels, + bias=True, + weight_init="zeros", + bias_init="zeros", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(self.norm(x)) + + +class PMFTransformer2DModel(ModelMixin, ConfigMixin): + """Native diffusers implementation of the pMF DiT backbone.""" + + _supports_gradient_checkpointing = True + _skip_layerwise_casting_patterns = ["pos_embed", "rope_freqs"] + + @register_to_config + def __init__( + self, + sample_size: int = 256, + patch_size: int = 16, + in_channels: int = 3, + hidden_size: int = 768, + depth: int = 16, + num_attention_heads: int = 12, + mlp_ratio: float = 8 / 3, + num_classes: int = 1000, + bottleneck_dim: int = 128, + aux_head_depth: int = 8, + num_class_tokens: int = 8, + num_time_tokens: int = 4, + num_cfg_tokens: int = 4, + num_interval_tokens: int = 2, + token_init_constant: float = 1.0, + embedding_init_constant: float = 1.0, + weight_init_constant: float = 0.32, + eval_mode: bool = True, + model_type: str | None = None, + num_class_embeds: int | None = None, + t_clip_min: float = 0.05, + norm_eps: float = 1e-6, + ): + super().__init__() + if num_class_embeds is not None: + num_classes = int(num_class_embeds) + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type in PMF_PRESET_CONFIGS: + preset = PMF_PRESET_CONFIGS[model_type] + sample_size = int(preset["sample_size"]) + patch_size = int(preset["patch_size"]) + hidden_size = int(preset["hidden_size"]) + depth = int(preset["depth"]) + num_attention_heads = int(preset["num_attention_heads"]) + bottleneck_dim = int(preset["bottleneck_dim"]) + aux_head_depth = int(preset["aux_head_depth"]) + + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.depth = depth + self.num_attention_heads = num_attention_heads + self.aux_head_depth = aux_head_depth + self.num_class_tokens = num_class_tokens + self.num_time_tokens = num_time_tokens + self.num_cfg_tokens = num_cfg_tokens + self.num_interval_tokens = num_interval_tokens + self.prefix_tokens = ( + num_class_tokens + num_cfg_tokens + 2 * num_interval_tokens + num_time_tokens + ) + self.t_clip_min = t_clip_min + self.eval_mode = eval_mode + self.gradient_checkpointing = False + + self.x_embedder = PMFBottleneckPatchEmbedder( + sample_size, + patch_size, + bottleneck_dim, + in_channels, + hidden_size, + bias=True, + ) + embed_kwargs = dict(hidden_size=hidden_size, init_constant=embedding_init_constant) + self.h_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.omega_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_start_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_end_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.y_embedder = PMFLabelEmbedder(num_classes, hidden_size, init_constant=embedding_init_constant) + + token_std = token_init_constant / math.sqrt(hidden_size) + self.time_tokens = nn.Parameter(torch.randn(1, num_time_tokens, hidden_size) * token_std) + self.class_tokens = nn.Parameter(torch.randn(1, num_class_tokens, hidden_size) * token_std) + self.omega_tokens = nn.Parameter(torch.randn(1, num_cfg_tokens, hidden_size) * token_std) + self.t_min_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + self.t_max_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + + total_tokens = self.x_embedder.num_patches + self.prefix_tokens + self.pos_embed = nn.Parameter(torch.randn(1, total_tokens, hidden_size) * 0.02) + + head_dim = hidden_size // num_attention_heads + self.register_buffer( + "rope_freqs", + precompute_rope_freqs(head_dim, self.x_embedder.num_patches), + persistent=False, + ) + + shared_depth = depth - aux_head_depth + block_kwargs = dict( + hidden_size=hidden_size, + num_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + weight_init_constant=weight_init_constant, + eps=norm_eps, + ) + self.shared_blocks = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(shared_depth)]) + self.u_heads = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth)]) + self.v_heads = nn.ModuleList( + [PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth if not eval_mode else 0)] + ) + self.u_final_layer = PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + self.v_final_layer = ( + PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + if not eval_mode + else None + ) + + def _build_sequence( + self, + sample: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + t_min: torch.Tensor, + t_max: torch.Tensor, + class_labels: torch.Tensor, + ) -> torch.Tensor: + x_embed = self.x_embedder(sample) + h_embed = self.h_embedder(h) + omega_embed = self.omega_embedder(1 - 1 / omega) + t_min_embed = self.cfg_t_start_embedder(t_min) + t_max_embed = self.cfg_t_end_embedder(t_max) + y_embed = self.y_embedder(class_labels) + + time_tokens = self.time_tokens + h_embed.unsqueeze(1) + omega_tokens = self.omega_tokens + omega_embed.unsqueeze(1) + t_min_tokens = self.t_min_tokens + t_min_embed.unsqueeze(1) + t_max_tokens = self.t_max_tokens + t_max_embed.unsqueeze(1) + class_tokens = self.class_tokens + y_embed.unsqueeze(1) + + seq = torch.cat( + [class_tokens, omega_tokens, t_min_tokens, t_max_tokens, time_tokens, x_embed], + dim=1, + ) + return seq + self.pos_embed + + def _unpatchify(self, tokens: torch.Tensor) -> torch.Tensor: + batch_size = tokens.shape[0] + patch = self.patch_size + grid = int(tokens.shape[1] ** 0.5) + channels = self.out_channels + x = tokens.reshape(batch_size, grid, grid, patch, patch, channels) + x = torch.einsum("nhwpqc->nchpwq", x) + return x.reshape(batch_size, channels, grid * patch, grid * patch) + + def forward( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: Optional[torch.Tensor] = None, + omega: Optional[torch.Tensor] = None, + guidance_interval_min: Optional[torch.Tensor] = None, + guidance_interval_max: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> PMFTransformer2DOutput | Tuple[torch.Tensor, Optional[torch.Tensor]]: + batch_size = sample.shape[0] + timestep = self._expand_batch(timestep, batch_size, sample.device, sample.dtype) + h = self._expand_batch(h if h is not None else timestep, batch_size, sample.device, sample.dtype) + omega = self._expand_batch( + omega if omega is not None else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_min = self._expand_batch( + guidance_interval_min + if guidance_interval_min is not None + else torch.zeros(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_max = self._expand_batch( + guidance_interval_max + if guidance_interval_max is not None + else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + + seq = self._build_sequence(sample, h, omega, guidance_interval_min, guidance_interval_max, class_labels) + rope_freqs = self.rope_freqs.to(device=sample.device) + + for block in self.shared_blocks: + if self.training and self.gradient_checkpointing: + seq = torch.utils.checkpoint.checkpoint(block, seq, rope_freqs, use_reentrant=False) + else: + seq = block(seq, rope_freqs) + + u_seq = v_seq = seq + for block in self.u_heads: + if self.training and self.gradient_checkpointing: + u_seq = torch.utils.checkpoint.checkpoint(block, u_seq, rope_freqs, use_reentrant=False) + else: + u_seq = block(u_seq, rope_freqs) + + for block in self.v_heads: + if self.training and self.gradient_checkpointing: + v_seq = torch.utils.checkpoint.checkpoint(block, v_seq, rope_freqs, use_reentrant=False) + else: + v_seq = block(v_seq, rope_freqs) + + u_tokens = u_seq[:, self.prefix_tokens :] + u_pred = self._unpatchify(self.u_final_layer(u_tokens)) + t = timestep.reshape(batch_size, 1, 1, 1) + u = (sample - u_pred) / torch.clamp(t, min=self.t_clip_min) + + v = None + if self.v_final_layer is not None: + v_tokens = v_seq[:, self.prefix_tokens :] + v_pred = self._unpatchify(self.v_final_layer(v_tokens)) + v = (sample - v_pred) / torch.clamp(t, min=self.t_clip_min) + + if not return_dict: + return (u, v) + return PMFTransformer2DOutput(u=u, v=v) + + @staticmethod + def _expand_batch( + value: torch.Tensor, + batch_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + value = torch.as_tensor(value, device=device, dtype=dtype) + if value.ndim == 0: + value = value.reshape(1) + if value.shape[0] == 1 and batch_size > 1: + value = value.expand(batch_size) + return value.reshape(batch_size) + + @classmethod + def from_pmf_checkpoint( + cls, + checkpoint_path: str, + model_type: str | None = None, + map_location: str = "cpu", + strict: bool = False, + ) -> Tuple["PMFTransformer2DModel", Dict[str, object]]: + checkpoint = torch.load(checkpoint_path, map_location=map_location, weights_only=False) + if isinstance(checkpoint, dict) and "state_dict" in checkpoint: + state_dict = checkpoint["state_dict"] + else: + state_dict = checkpoint + + if model_type is None: + for key in ("model_type", "model_str", "model"): + if isinstance(checkpoint, dict) and key in checkpoint: + model_type = checkpoint[key] + break + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type is None: + raise ValueError("model_type is required when it cannot be inferred from the checkpoint.") + + config = dict(PMF_PRESET_CONFIGS[model_type]) + config["model_type"] = model_type + config["eval_mode"] = True + model = cls(**config) + model.load_state_dict(remap_legacy_state_dict(state_dict), strict=strict) + metadata = {"checkpoint_path": checkpoint_path, "model_type": model_type} + return model, metadata + + def to_pmf_checkpoint(self, prefix: str = "net.") -> Dict[str, torch.Tensor]: + state_dict: Dict[str, torch.Tensor] = {} + for key, value in self.state_dict().items(): + state_dict[f"{prefix}{key}"] = value.detach().cpu() + return state_dict + + @property + def net(self): + return self + + +PMFDiffusersModel = PMFTransformer2DModel diff --git a/pMF-L-32-SIM/README.md b/pMF-L-32-SIM/README.md new file mode 100644 index 0000000000000000000000000000000000000000..84559869aa5a37191264df464b94b1824af0d0fe --- /dev/null +++ b/pMF-L-32-SIM/README.md @@ -0,0 +1,44 @@ +--- +license: mit +library_name: diffusers +pipeline_tag: text-to-image +tags: +- diffusers +- pmf +- image-generation +- class-conditional +- imagenet +inference: true +--- + +# pMF-L-32-SIM + +Self-contained Diffusers variant for **pMF-L/32 (FD-SIM post-trained)** (Pixel Mean Flows). + +Recommended settings: `guidance_scale=7.5`, interval `[0.2, 0.6]`, `noise_scale=4.0`. + +## Load + +```python +from pathlib import Path +from diffusers import DiffusionPipeline +import torch + +model_dir = Path("./pMF-L-32-SIM") +pipe = DiffusionPipeline.from_pretrained( + str(model_dir), + local_files_only=True, + custom_pipeline=str(model_dir / "pipeline.py"), + trust_remote_code=True, + torch_dtype=torch.bfloat16, +).to("cuda") + +image = pipe( + class_labels=207, + num_inference_steps=1, + guidance_scale=7.5, + guidance_interval_min=0.2, + guidance_interval_max=0.6, + noise_scale=4.0, +).images[0] +``` diff --git a/pMF-L-32-SIM/model_index.json b/pMF-L-32-SIM/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..ec00b5bd38c72b4310e6e3cb2c65ce31ef2427b4 --- /dev/null +++ b/pMF-L-32-SIM/model_index.json @@ -0,0 +1,1017 @@ +{ + "_class_name": [ + "pipeline", + "PMFPipeline" + ], + "_diffusers_version": "0.36.0", + "scheduler": [ + "scheduling_pmf", + "PMFScheduler" + ], + "transformer": [ + "transformer_pmf", + "PMFTransformer2DModel" + ], + "id2label": { + "0": "tench, Tinca tinca", + "1": "goldfish, Carassius auratus", + "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "3": "tiger shark, Galeocerdo cuvieri", + "4": "hammerhead, hammerhead shark", + "5": "electric ray, crampfish, numbfish, torpedo", + "6": "stingray", + "7": "cock", + "8": "hen", + "9": "ostrich, Struthio camelus", + "10": "brambling, Fringilla montifringilla", + "11": "goldfinch, Carduelis carduelis", + "12": "house finch, linnet, Carpodacus mexicanus", + "13": "junco, snowbird", + "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "15": "robin, American robin, Turdus migratorius", + "16": "bulbul", + "17": "jay", + "18": "magpie", + "19": "chickadee", + "20": "water ouzel, dipper", + "21": "kite", + "22": "bald eagle, American eagle, Haliaeetus leucocephalus", + "23": "vulture", + "24": "great grey owl, great gray owl, Strix nebulosa", + "25": "European fire salamander, Salamandra salamandra", + "26": "common newt, Triturus vulgaris", + "27": "eft", + "28": "spotted salamander, Ambystoma maculatum", + "29": "axolotl, mud puppy, Ambystoma mexicanum", + "30": "bullfrog, Rana catesbeiana", + "31": "tree frog, tree-frog", + "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "33": "loggerhead, loggerhead turtle, Caretta caretta", + "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "35": "mud turtle", + "36": "terrapin", + "37": "box turtle, box tortoise", + "38": "banded gecko", + "39": "common iguana, iguana, Iguana iguana", + "40": "American chameleon, anole, Anolis carolinensis", + "41": "whiptail, whiptail lizard", + "42": "agama", + "43": "frilled lizard, Chlamydosaurus kingi", + "44": "alligator lizard", + "45": "Gila monster, Heloderma suspectum", + "46": "green lizard, Lacerta viridis", + "47": "African chameleon, Chamaeleo chamaeleon", + "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "49": "African crocodile, Nile crocodile, Crocodylus niloticus", + "50": "American alligator, Alligator mississipiensis", + "51": "triceratops", + "52": "thunder snake, worm snake, Carphophis amoenus", + "53": "ringneck snake, ring-necked snake, ring snake", + "54": "hognose snake, puff adder, sand viper", + "55": "green snake, grass snake", + "56": "king snake, kingsnake", + "57": "garter snake, grass snake", + "58": "water snake", + "59": "vine snake", + "60": "night snake, Hypsiglena torquata", + "61": "boa constrictor, Constrictor constrictor", + "62": "rock python, rock snake, Python sebae", + "63": "Indian cobra, Naja naja", + "64": "green mamba", + "65": "sea snake", + "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "68": "sidewinder, horned rattlesnake, Crotalus cerastes", + "69": "trilobite", + "70": "harvestman, daddy longlegs, Phalangium opilio", + "71": "scorpion", + "72": "black and gold garden spider, Argiope aurantia", + "73": "barn spider, Araneus cavaticus", + "74": "garden spider, Aranea diademata", + "75": "black widow, Latrodectus mactans", + "76": "tarantula", + "77": "wolf spider, hunting spider", + "78": "tick", + "79": "centipede", + "80": "black grouse", + "81": "ptarmigan", + "82": "ruffed grouse, partridge, Bonasa umbellus", + "83": "prairie chicken, prairie grouse, prairie fowl", + "84": "peacock", + "85": "quail", + "86": "partridge", + "87": "African grey, African gray, Psittacus erithacus", + "88": "macaw", + "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "90": "lorikeet", + "91": "coucal", + "92": "bee eater", + "93": "hornbill", + "94": "hummingbird", + "95": "jacamar", + "96": "toucan", + "97": "drake", + "98": "red-breasted merganser, Mergus serrator", + "99": "goose", + "100": "black swan, Cygnus atratus", + "101": "tusker", + "102": "echidna, spiny anteater, anteater", + "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "104": "wallaby, brush kangaroo", + "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "106": "wombat", + "107": "jellyfish", + "108": "sea anemone, anemone", + "109": "brain coral", + "110": "flatworm, platyhelminth", + "111": "nematode, nematode worm, roundworm", + "112": "conch", + "113": "snail", + "114": "slug", + "115": "sea slug, nudibranch", + "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "117": "chambered nautilus, pearly nautilus, nautilus", + "118": "Dungeness crab, Cancer magister", + "119": "rock crab, Cancer irroratus", + "120": "fiddler crab", + "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "124": "crayfish, crawfish, crawdad, crawdaddy", + "125": "hermit crab", + "126": "isopod", + "127": "white stork, Ciconia ciconia", + "128": "black stork, Ciconia nigra", + "129": "spoonbill", + "130": "flamingo", + "131": "little blue heron, Egretta caerulea", + "132": "American egret, great white heron, Egretta albus", + "133": "bittern", + "134": "crane", + "135": "limpkin, Aramus pictus", + "136": "European gallinule, Porphyrio porphyrio", + "137": "American coot, marsh hen, mud hen, water hen, Fulica americana", + "138": "bustard", + "139": "ruddy turnstone, Arenaria interpres", + "140": "red-backed sandpiper, dunlin, Erolia alpina", + "141": "redshank, Tringa totanus", + "142": "dowitcher", + "143": "oystercatcher, oyster catcher", + "144": "pelican", + "145": "king penguin, Aptenodytes patagonica", + "146": "albatross, mollymawk", + "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "149": "dugong, Dugong dugon", + "150": "sea lion", + "151": "Chihuahua", + "152": "Japanese spaniel", + "153": "Maltese dog, Maltese terrier, Maltese", + "154": "Pekinese, Pekingese, Peke", + "155": "Shih-Tzu", + "156": "Blenheim spaniel", + "157": "papillon", + "158": "toy terrier", + "159": "Rhodesian ridgeback", + "160": "Afghan hound, Afghan", + "161": "basset, basset hound", + "162": "beagle", + "163": "bloodhound, sleuthhound", + "164": "bluetick", + "165": "black-and-tan coonhound", + "166": "Walker hound, Walker foxhound", + "167": "English foxhound", + "168": "redbone", + "169": "borzoi, Russian wolfhound", + "170": "Irish wolfhound", + "171": "Italian greyhound", + "172": "whippet", + "173": "Ibizan hound, Ibizan Podenco", + "174": "Norwegian elkhound, elkhound", + "175": "otterhound, otter hound", + "176": "Saluki, gazelle hound", + "177": "Scottish deerhound, deerhound", + "178": "Weimaraner", + "179": "Staffordshire bullterrier, Staffordshire bull terrier", + "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "181": "Bedlington terrier", + "182": "Border terrier", + "183": "Kerry blue terrier", + "184": "Irish terrier", + "185": "Norfolk terrier", + "186": "Norwich terrier", + "187": "Yorkshire terrier", + "188": "wire-haired fox terrier", + "189": "Lakeland terrier", + "190": "Sealyham terrier, Sealyham", + "191": "Airedale, Airedale terrier", + "192": "cairn, cairn terrier", + "193": "Australian terrier", + "194": "Dandie Dinmont, Dandie Dinmont terrier", + "195": "Boston bull, Boston terrier", + "196": "miniature schnauzer", + "197": "giant schnauzer", + "198": "standard schnauzer", + "199": "Scotch terrier, Scottish terrier, Scottie", + "200": "Tibetan terrier, chrysanthemum dog", + "201": "silky terrier, Sydney silky", + "202": "soft-coated wheaten terrier", + "203": "West Highland white terrier", + "204": "Lhasa, Lhasa apso", + "205": "flat-coated retriever", + "206": "curly-coated retriever", + "207": "golden retriever", + "208": "Labrador retriever", + "209": "Chesapeake Bay retriever", + "210": "German short-haired pointer", + "211": "vizsla, Hungarian pointer", + "212": "English setter", + "213": "Irish setter, red setter", + "214": "Gordon setter", + "215": "Brittany spaniel", + "216": "clumber, clumber spaniel", + "217": "English springer, English springer spaniel", + "218": "Welsh springer spaniel", + "219": "cocker spaniel, English cocker spaniel, cocker", + "220": "Sussex spaniel", + "221": "Irish water spaniel", + "222": "kuvasz", + "223": "schipperke", + "224": "groenendael", + "225": "malinois", + "226": "briard", + "227": "kelpie", + "228": "komondor", + "229": "Old English sheepdog, bobtail", + "230": "Shetland sheepdog, Shetland sheep dog, Shetland", + "231": "collie", + "232": "Border collie", + "233": "Bouvier des Flandres, Bouviers des Flandres", + "234": "Rottweiler", + "235": "German shepherd, German shepherd dog, German police dog, alsatian", + "236": "Doberman, Doberman pinscher", + "237": "miniature pinscher", + "238": "Greater Swiss Mountain dog", + "239": "Bernese mountain dog", + "240": "Appenzeller", + "241": "EntleBucher", + "242": "boxer", + "243": "bull mastiff", + "244": "Tibetan mastiff", + "245": "French bulldog", + "246": "Great Dane", + "247": "Saint Bernard, St Bernard", + "248": "Eskimo dog, husky", + "249": "malamute, malemute, Alaskan malamute", + "250": "Siberian husky", + "251": "dalmatian, coach dog, carriage dog", + "252": "affenpinscher, monkey pinscher, monkey dog", + "253": "basenji", + "254": "pug, pug-dog", + "255": "Leonberg", + "256": "Newfoundland, Newfoundland dog", + "257": "Great Pyrenees", + "258": "Samoyed, Samoyede", + "259": "Pomeranian", + "260": "chow, chow chow", + "261": "keeshond", + "262": "Brabancon griffon", + "263": "Pembroke, Pembroke Welsh corgi", + "264": "Cardigan, Cardigan Welsh corgi", + "265": "toy poodle", + "266": "miniature poodle", + "267": "standard poodle", + "268": "Mexican hairless", + "269": "timber wolf, grey wolf, gray wolf, Canis lupus", + "270": "white wolf, Arctic wolf, Canis lupus tundrarum", + "271": "red wolf, maned wolf, Canis rufus, Canis niger", + "272": "coyote, prairie wolf, brush wolf, Canis latrans", + "273": "dingo, warrigal, warragal, Canis dingo", + "274": "dhole, Cuon alpinus", + "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "276": "hyena, hyaena", + "277": "red fox, Vulpes vulpes", + "278": "kit fox, Vulpes macrotis", + "279": "Arctic fox, white fox, Alopex lagopus", + "280": "grey fox, gray fox, Urocyon cinereoargenteus", + "281": "tabby, tabby cat", + "282": "tiger cat", + "283": "Persian cat", + "284": "Siamese cat, Siamese", + "285": "Egyptian cat", + "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "287": "lynx, catamount", + "288": "leopard, Panthera pardus", + "289": "snow leopard, ounce, Panthera uncia", + "290": "jaguar, panther, Panthera onca, Felis onca", + "291": "lion, king of beasts, Panthera leo", + "292": "tiger, Panthera tigris", + "293": "cheetah, chetah, Acinonyx jubatus", + "294": "brown bear, bruin, Ursus arctos", + "295": "American black bear, black bear, Ursus americanus, Euarctos americanus", + "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "297": "sloth bear, Melursus ursinus, Ursus ursinus", + "298": "mongoose", + "299": "meerkat, mierkat", + "300": "tiger beetle", + "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "302": "ground beetle, carabid beetle", + "303": "long-horned beetle, longicorn, longicorn beetle", + "304": "leaf beetle, chrysomelid", + "305": "dung beetle", + "306": "rhinoceros beetle", + "307": "weevil", + "308": "fly", + "309": "bee", + "310": "ant, emmet, pismire", + "311": "grasshopper, hopper", + "312": "cricket", + "313": "walking stick, walkingstick, stick insect", + "314": "cockroach, roach", + "315": "mantis, mantid", + "316": "cicada, cicala", + "317": "leafhopper", + "318": "lacewing, lacewing fly", + "319": "dragonfly, darning needle, devils darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "320": "damselfly", + "321": "admiral", + "322": "ringlet, ringlet butterfly", + "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "324": "cabbage butterfly", + "325": "sulphur butterfly, sulfur butterfly", + "326": "lycaenid, lycaenid butterfly", + "327": "starfish, sea star", + "328": "sea urchin", + "329": "sea cucumber, holothurian", + "330": "wood rabbit, cottontail, cottontail rabbit", + "331": "hare", + "332": "Angora, Angora rabbit", + "333": "hamster", + "334": "porcupine, hedgehog", + "335": "fox squirrel, eastern fox squirrel, Sciurus niger", + "336": "marmot", + "337": "beaver", + "338": "guinea pig, Cavia cobaya", + "339": "sorrel", + "340": "zebra", + "341": "hog, pig, grunter, squealer, Sus scrofa", + "342": "wild boar, boar, Sus scrofa", + "343": "warthog", + "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "345": "ox", + "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "347": "bison", + "348": "ram, tup", + "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "350": "ibex, Capra ibex", + "351": "hartebeest", + "352": "impala, Aepyceros melampus", + "353": "gazelle", + "354": "Arabian camel, dromedary, Camelus dromedarius", + "355": "llama", + "356": "weasel", + "357": "mink", + "358": "polecat, fitch, foulmart, foumart, Mustela putorius", + "359": "black-footed ferret, ferret, Mustela nigripes", + "360": "otter", + "361": "skunk, polecat, wood pussy", + "362": "badger", + "363": "armadillo", + "364": "three-toed sloth, ai, Bradypus tridactylus", + "365": "orangutan, orang, orangutang, Pongo pygmaeus", + "366": "gorilla, Gorilla gorilla", + "367": "chimpanzee, chimp, Pan troglodytes", + "368": "gibbon, Hylobates lar", + "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "370": "guenon, guenon monkey", + "371": "patas, hussar monkey, Erythrocebus patas", + "372": "baboon", + "373": "macaque", + "374": "langur", + "375": "colobus, colobus monkey", + "376": "proboscis monkey, Nasalis larvatus", + "377": "marmoset", + "378": "capuchin, ringtail, Cebus capucinus", + "379": "howler monkey, howler", + "380": "titi, titi monkey", + "381": "spider monkey, Ateles geoffroyi", + "382": "squirrel monkey, Saimiri sciureus", + "383": "Madagascar cat, ring-tailed lemur, Lemur catta", + "384": "indri, indris, Indri indri, Indri brevicaudatus", + "385": "Indian elephant, Elephas maximus", + "386": "African elephant, Loxodonta africana", + "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "389": "barracouta, snoek", + "390": "eel", + "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "392": "rock beauty, Holocanthus tricolor", + "393": "anemone fish", + "394": "sturgeon", + "395": "gar, garfish, garpike, billfish, Lepisosteus osseus", + "396": "lionfish", + "397": "puffer, pufferfish, blowfish, globefish", + "398": "abacus", + "399": "abaya", + "400": "academic gown, academic robe, judge robe", + "401": "accordion, piano accordion, squeeze box", + "402": "acoustic guitar", + "403": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "404": "airliner", + "405": "airship, dirigible", + "406": "altar", + "407": "ambulance", + "408": "amphibian, amphibious vehicle", + "409": "analog clock", + "410": "apiary, bee house", + "411": "apron", + "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "413": "assault rifle, assault gun", + "414": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "415": "bakery, bakeshop, bakehouse", + "416": "balance beam, beam", + "417": "balloon", + "418": "ballpoint, ballpoint pen, ballpen, Biro", + "419": "Band Aid", + "420": "banjo", + "421": "bannister, banister, balustrade, balusters, handrail", + "422": "barbell", + "423": "barber chair", + "424": "barbershop", + "425": "barn", + "426": "barometer", + "427": "barrel, cask", + "428": "barrow, garden cart, lawn cart, wheelbarrow", + "429": "baseball", + "430": "basketball", + "431": "bassinet", + "432": "bassoon", + "433": "bathing cap, swimming cap", + "434": "bath towel", + "435": "bathtub, bathing tub, bath, tub", + "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "437": "beacon, lighthouse, beacon light, pharos", + "438": "beaker", + "439": "bearskin, busby, shako", + "440": "beer bottle", + "441": "beer glass", + "442": "bell cote, bell cot", + "443": "bib", + "444": "bicycle-built-for-two, tandem bicycle, tandem", + "445": "bikini, two-piece", + "446": "binder, ring-binder", + "447": "binoculars, field glasses, opera glasses", + "448": "birdhouse", + "449": "boathouse", + "450": "bobsled, bobsleigh, bob", + "451": "bolo tie, bolo, bola tie, bola", + "452": "bonnet, poke bonnet", + "453": "bookcase", + "454": "bookshop, bookstore, bookstall", + "455": "bottlecap", + "456": "bow", + "457": "bow tie, bow-tie, bowtie", + "458": "brass, memorial tablet, plaque", + "459": "brassiere, bra, bandeau", + "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "461": "breastplate, aegis, egis", + "462": "broom", + "463": "bucket, pail", + "464": "buckle", + "465": "bulletproof vest", + "466": "bullet train, bullet", + "467": "butcher shop, meat market", + "468": "cab, hack, taxi, taxicab", + "469": "caldron, cauldron", + "470": "candle, taper, wax light", + "471": "cannon", + "472": "canoe", + "473": "can opener, tin opener", + "474": "cardigan", + "475": "car mirror", + "476": "carousel, carrousel, merry-go-round, roundabout, whirligig", + "477": "carpenters kit, tool kit", + "478": "carton", + "479": "car wheel", + "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "481": "cassette", + "482": "cassette player", + "483": "castle", + "484": "catamaran", + "485": "CD player", + "486": "cello, violoncello", + "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "488": "chain", + "489": "chainlink fence", + "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "491": "chain saw, chainsaw", + "492": "chest", + "493": "chiffonier, commode", + "494": "chime, bell, gong", + "495": "china cabinet, china closet", + "496": "Christmas stocking", + "497": "church, church building", + "498": "cinema, movie theater, movie theatre, movie house, picture palace", + "499": "cleaver, meat cleaver, chopper", + "500": "cliff dwelling", + "501": "cloak", + "502": "clog, geta, patten, sabot", + "503": "cocktail shaker", + "504": "coffee mug", + "505": "coffeepot", + "506": "coil, spiral, volute, whorl, helix", + "507": "combination lock", + "508": "computer keyboard, keypad", + "509": "confectionery, confectionary, candy store", + "510": "container ship, containership, container vessel", + "511": "convertible", + "512": "corkscrew, bottle screw", + "513": "cornet, horn, trumpet, trump", + "514": "cowboy boot", + "515": "cowboy hat, ten-gallon hat", + "516": "cradle", + "517": "crane", + "518": "crash helmet", + "519": "crate", + "520": "crib, cot", + "521": "Crock Pot", + "522": "croquet ball", + "523": "crutch", + "524": "cuirass", + "525": "dam, dike, dyke", + "526": "desk", + "527": "desktop computer", + "528": "dial telephone, dial phone", + "529": "diaper, nappy, napkin", + "530": "digital clock", + "531": "digital watch", + "532": "dining table, board", + "533": "dishrag, dishcloth", + "534": "dishwasher, dish washer, dishwashing machine", + "535": "disk brake, disc brake", + "536": "dock, dockage, docking facility", + "537": "dogsled, dog sled, dog sleigh", + "538": "dome", + "539": "doormat, welcome mat", + "540": "drilling platform, offshore rig", + "541": "drum, membranophone, tympan", + "542": "drumstick", + "543": "dumbbell", + "544": "Dutch oven", + "545": "electric fan, blower", + "546": "electric guitar", + "547": "electric locomotive", + "548": "entertainment center", + "549": "envelope", + "550": "espresso maker", + "551": "face powder", + "552": "feather boa, boa", + "553": "file, file cabinet, filing cabinet", + "554": "fireboat", + "555": "fire engine, fire truck", + "556": "fire screen, fireguard", + "557": "flagpole, flagstaff", + "558": "flute, transverse flute", + "559": "folding chair", + "560": "football helmet", + "561": "forklift", + "562": "fountain", + "563": "fountain pen", + "564": "four-poster", + "565": "freight car", + "566": "French horn, horn", + "567": "frying pan, frypan, skillet", + "568": "fur coat", + "569": "garbage truck, dustcart", + "570": "gasmask, respirator, gas helmet", + "571": "gas pump, gasoline pump, petrol pump, island dispenser", + "572": "goblet", + "573": "go-kart", + "574": "golf ball", + "575": "golfcart, golf cart", + "576": "gondola", + "577": "gong, tam-tam", + "578": "gown", + "579": "grand piano, grand", + "580": "greenhouse, nursery, glasshouse", + "581": "grille, radiator grille", + "582": "grocery store, grocery, food market, market", + "583": "guillotine", + "584": "hair slide", + "585": "hair spray", + "586": "half track", + "587": "hammer", + "588": "hamper", + "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "590": "hand-held computer, hand-held microcomputer", + "591": "handkerchief, hankie, hanky, hankey", + "592": "hard disc, hard disk, fixed disk", + "593": "harmonica, mouth organ, harp, mouth harp", + "594": "harp", + "595": "harvester, reaper", + "596": "hatchet", + "597": "holster", + "598": "home theater, home theatre", + "599": "honeycomb", + "600": "hook, claw", + "601": "hoopskirt, crinoline", + "602": "horizontal bar, high bar", + "603": "horse cart, horse-cart", + "604": "hourglass", + "605": "iPod", + "606": "iron, smoothing iron", + "607": "jack-o-lantern", + "608": "jean, blue jean, denim", + "609": "jeep, landrover", + "610": "jersey, T-shirt, tee shirt", + "611": "jigsaw puzzle", + "612": "jinrikisha, ricksha, rickshaw", + "613": "joystick", + "614": "kimono", + "615": "knee pad", + "616": "knot", + "617": "lab coat, laboratory coat", + "618": "ladle", + "619": "lampshade, lamp shade", + "620": "laptop, laptop computer", + "621": "lawn mower, mower", + "622": "lens cap, lens cover", + "623": "letter opener, paper knife, paperknife", + "624": "library", + "625": "lifeboat", + "626": "lighter, light, igniter, ignitor", + "627": "limousine, limo", + "628": "liner, ocean liner", + "629": "lipstick, lip rouge", + "630": "Loafer", + "631": "lotion", + "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "633": "loupe, jewelers loupe", + "634": "lumbermill, sawmill", + "635": "magnetic compass", + "636": "mailbag, postbag", + "637": "mailbox, letter box", + "638": "maillot", + "639": "maillot, tank suit", + "640": "manhole cover", + "641": "maraca", + "642": "marimba, xylophone", + "643": "mask", + "644": "matchstick", + "645": "maypole", + "646": "maze, labyrinth", + "647": "measuring cup", + "648": "medicine chest, medicine cabinet", + "649": "megalith, megalithic structure", + "650": "microphone, mike", + "651": "microwave, microwave oven", + "652": "military uniform", + "653": "milk can", + "654": "minibus", + "655": "miniskirt, mini", + "656": "minivan", + "657": "missile", + "658": "mitten", + "659": "mixing bowl", + "660": "mobile home, manufactured home", + "661": "Model T", + "662": "modem", + "663": "monastery", + "664": "monitor", + "665": "moped", + "666": "mortar", + "667": "mortarboard", + "668": "mosque", + "669": "mosquito net", + "670": "motor scooter, scooter", + "671": "mountain bike, all-terrain bike, off-roader", + "672": "mountain tent", + "673": "mouse, computer mouse", + "674": "mousetrap", + "675": "moving van", + "676": "muzzle", + "677": "nail", + "678": "neck brace", + "679": "necklace", + "680": "nipple", + "681": "notebook, notebook computer", + "682": "obelisk", + "683": "oboe, hautboy, hautbois", + "684": "ocarina, sweet potato", + "685": "odometer, hodometer, mileometer, milometer", + "686": "oil filter", + "687": "organ, pipe organ", + "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "689": "overskirt", + "690": "oxcart", + "691": "oxygen mask", + "692": "packet", + "693": "paddle, boat paddle", + "694": "paddlewheel, paddle wheel", + "695": "padlock", + "696": "paintbrush", + "697": "pajama, pyjama, pjs, jammies", + "698": "palace", + "699": "panpipe, pandean pipe, syrinx", + "700": "paper towel", + "701": "parachute, chute", + "702": "parallel bars, bars", + "703": "park bench", + "704": "parking meter", + "705": "passenger car, coach, carriage", + "706": "patio, terrace", + "707": "pay-phone, pay-station", + "708": "pedestal, plinth, footstall", + "709": "pencil box, pencil case", + "710": "pencil sharpener", + "711": "perfume, essence", + "712": "Petri dish", + "713": "photocopier", + "714": "pick, plectrum, plectron", + "715": "pickelhaube", + "716": "picket fence, paling", + "717": "pickup, pickup truck", + "718": "pier", + "719": "piggy bank, penny bank", + "720": "pill bottle", + "721": "pillow", + "722": "ping-pong ball", + "723": "pinwheel", + "724": "pirate, pirate ship", + "725": "pitcher, ewer", + "726": "plane, carpenters plane, woodworking plane", + "727": "planetarium", + "728": "plastic bag", + "729": "plate rack", + "730": "plow, plough", + "731": "plunger, plumbers helper", + "732": "Polaroid camera, Polaroid Land camera", + "733": "pole", + "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "735": "poncho", + "736": "pool table, billiard table, snooker table", + "737": "pop bottle, soda bottle", + "738": "pot, flowerpot", + "739": "potters wheel", + "740": "power drill", + "741": "prayer rug, prayer mat", + "742": "printer", + "743": "prison, prison house", + "744": "projectile, missile", + "745": "projector", + "746": "puck, hockey puck", + "747": "punching bag, punch bag, punching ball, punchball", + "748": "purse", + "749": "quill, quill pen", + "750": "quilt, comforter, comfort, puff", + "751": "racer, race car, racing car", + "752": "racket, racquet", + "753": "radiator", + "754": "radio, wireless", + "755": "radio telescope, radio reflector", + "756": "rain barrel", + "757": "recreational vehicle, RV, R.V.", + "758": "reel", + "759": "reflex camera", + "760": "refrigerator, icebox", + "761": "remote control, remote", + "762": "restaurant, eating house, eating place, eatery", + "763": "revolver, six-gun, six-shooter", + "764": "rifle", + "765": "rocking chair, rocker", + "766": "rotisserie", + "767": "rubber eraser, rubber, pencil eraser", + "768": "rugby ball", + "769": "rule, ruler", + "770": "running shoe", + "771": "safe", + "772": "safety pin", + "773": "saltshaker, salt shaker", + "774": "sandal", + "775": "sarong", + "776": "sax, saxophone", + "777": "scabbard", + "778": "scale, weighing machine", + "779": "school bus", + "780": "schooner", + "781": "scoreboard", + "782": "screen, CRT screen", + "783": "screw", + "784": "screwdriver", + "785": "seat belt, seatbelt", + "786": "sewing machine", + "787": "shield, buckler", + "788": "shoe shop, shoe-shop, shoe store", + "789": "shoji", + "790": "shopping basket", + "791": "shopping cart", + "792": "shovel", + "793": "shower cap", + "794": "shower curtain", + "795": "ski", + "796": "ski mask", + "797": "sleeping bag", + "798": "slide rule, slipstick", + "799": "sliding door", + "800": "slot, one-armed bandit", + "801": "snorkel", + "802": "snowmobile", + "803": "snowplow, snowplough", + "804": "soap dispenser", + "805": "soccer ball", + "806": "sock", + "807": "solar dish, solar collector, solar furnace", + "808": "sombrero", + "809": "soup bowl", + "810": "space bar", + "811": "space heater", + "812": "space shuttle", + "813": "spatula", + "814": "speedboat", + "815": "spider web, spiders web", + "816": "spindle", + "817": "sports car, sport car", + "818": "spotlight, spot", + "819": "stage", + "820": "steam locomotive", + "821": "steel arch bridge", + "822": "steel drum", + "823": "stethoscope", + "824": "stole", + "825": "stone wall", + "826": "stopwatch, stop watch", + "827": "stove", + "828": "strainer", + "829": "streetcar, tram, tramcar, trolley, trolley car", + "830": "stretcher", + "831": "studio couch, day bed", + "832": "stupa, tope", + "833": "submarine, pigboat, sub, U-boat", + "834": "suit, suit of clothes", + "835": "sundial", + "836": "sunglass", + "837": "sunglasses, dark glasses, shades", + "838": "sunscreen, sunblock, sun blocker", + "839": "suspension bridge", + "840": "swab, swob, mop", + "841": "sweatshirt", + "842": "swimming trunks, bathing trunks", + "843": "swing", + "844": "switch, electric switch, electrical switch", + "845": "syringe", + "846": "table lamp", + "847": "tank, army tank, armored combat vehicle, armoured combat vehicle", + "848": "tape player", + "849": "teapot", + "850": "teddy, teddy bear", + "851": "television, television system", + "852": "tennis ball", + "853": "thatch, thatched roof", + "854": "theater curtain, theatre curtain", + "855": "thimble", + "856": "thresher, thrasher, threshing machine", + "857": "throne", + "858": "tile roof", + "859": "toaster", + "860": "tobacco shop, tobacconist shop, tobacconist", + "861": "toilet seat", + "862": "torch", + "863": "totem pole", + "864": "tow truck, tow car, wrecker", + "865": "toyshop", + "866": "tractor", + "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "868": "tray", + "869": "trench coat", + "870": "tricycle, trike, velocipede", + "871": "trimaran", + "872": "tripod", + "873": "triumphal arch", + "874": "trolleybus, trolley coach, trackless trolley", + "875": "trombone", + "876": "tub, vat", + "877": "turnstile", + "878": "typewriter keyboard", + "879": "umbrella", + "880": "unicycle, monocycle", + "881": "upright, upright piano", + "882": "vacuum, vacuum cleaner", + "883": "vase", + "884": "vault", + "885": "velvet", + "886": "vending machine", + "887": "vestment", + "888": "viaduct", + "889": "violin, fiddle", + "890": "volleyball", + "891": "waffle iron", + "892": "wall clock", + "893": "wallet, billfold, notecase, pocketbook", + "894": "wardrobe, closet, press", + "895": "warplane, military plane", + "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "897": "washer, automatic washer, washing machine", + "898": "water bottle", + "899": "water jug", + "900": "water tower", + "901": "whiskey jug", + "902": "whistle", + "903": "wig", + "904": "window screen", + "905": "window shade", + "906": "Windsor tie", + "907": "wine bottle", + "908": "wing", + "909": "wok", + "910": "wooden spoon", + "911": "wool, woolen, woollen", + "912": "worm fence, snake fence, snake-rail fence, Virginia fence", + "913": "wreck", + "914": "yawl", + "915": "yurt", + "916": "web site, website, internet site, site", + "917": "comic book", + "918": "crossword puzzle, crossword", + "919": "street sign", + "920": "traffic light, traffic signal, stoplight", + "921": "book jacket, dust cover, dust jacket, dust wrapper", + "922": "menu", + "923": "plate", + "924": "guacamole", + "925": "consomme", + "926": "hot pot, hotpot", + "927": "trifle", + "928": "ice cream, icecream", + "929": "ice lolly, lolly, lollipop, popsicle", + "930": "French loaf", + "931": "bagel, beigel", + "932": "pretzel", + "933": "cheeseburger", + "934": "hotdog, hot dog, red hot", + "935": "mashed potato", + "936": "head cabbage", + "937": "broccoli", + "938": "cauliflower", + "939": "zucchini, courgette", + "940": "spaghetti squash", + "941": "acorn squash", + "942": "butternut squash", + "943": "cucumber, cuke", + "944": "artichoke, globe artichoke", + "945": "bell pepper", + "946": "cardoon", + "947": "mushroom", + "948": "Granny Smith", + "949": "strawberry", + "950": "orange", + "951": "lemon", + "952": "fig", + "953": "pineapple, ananas", + "954": "banana", + "955": "jackfruit, jak, jack", + "956": "custard apple", + "957": "pomegranate", + "958": "hay", + "959": "carbonara", + "960": "chocolate sauce, chocolate syrup", + "961": "dough", + "962": "meat loaf, meatloaf", + "963": "pizza, pizza pie", + "964": "potpie", + "965": "burrito", + "966": "red wine", + "967": "espresso", + "968": "cup", + "969": "eggnog", + "970": "alp", + "971": "bubble", + "972": "cliff, drop, drop-off", + "973": "coral reef", + "974": "geyser", + "975": "lakeside, lakeshore", + "976": "promontory, headland, head, foreland", + "977": "sandbar, sand bar", + "978": "seashore, coast, seacoast, sea-coast", + "979": "valley, vale", + "980": "volcano", + "981": "ballplayer, baseball player", + "982": "groom, bridegroom", + "983": "scuba diver", + "984": "rapeseed", + "985": "daisy", + "986": "yellow ladys slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "987": "corn", + "988": "acorn", + "989": "hip, rose hip, rosehip", + "990": "buckeye, horse chestnut, conker", + "991": "coral fungus", + "992": "agaric", + "993": "gyromitra", + "994": "stinkhorn, carrion fungus", + "995": "earthstar", + "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "997": "bolete", + "998": "ear, spike, capitulum", + "999": "toilet tissue, toilet paper, bathroom tissue" + } +} diff --git a/pMF-L-32-SIM/pipeline.py b/pMF-L-32-SIM/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..314f675b581fe55ede8cbc0d1f2e1f850e5e9f34 --- /dev/null +++ b/pMF-L-32-SIM/pipeline.py @@ -0,0 +1,289 @@ +"""Hub custom pipeline: PMFPipeline.""" + +from __future__ import annotations + +import inspect +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from diffusers.utils.torch_utils import randn_tensor + +DEFAULT_CFG_BY_MODEL: Dict[str, Dict[str, float]] = { + "pMF-B/16": {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8}, + "pMF-B/32": {"guidance_scale": 6.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.7}, + "pMF-L/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.7}, + "pMF-L/32": {"guidance_scale": 7.5, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/16": {"guidance_scale": 7.0, "guidance_interval_min": 0.2, "guidance_interval_max": 0.6}, + "pMF-H/32": {"guidance_scale": 5.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.6}, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + + +class PMFPipeline(DiffusionPipeline): + model_cpu_offload_seq = "transformer" + + def __init__( + self, + transformer: Any, + scheduler: Any, + id2label: Optional[Dict[Union[int, str], str]] = None, + ) -> None: + super().__init__() + if scheduler is None: + raise ValueError("PMFPipeline requires a scheduler loaded from the checkpoint.") + self.register_modules(transformer=transformer, scheduler=scheduler) + self._id2label = self._normalize_id2label(id2label) + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = bool(self._id2label) + + @staticmethod + def prepare_extra_step_kwargs( + scheduler: Any, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + eta: float | None = None, + ) -> Dict[str, Any]: + kwargs: Dict[str, Any] = {} + step_params = set(inspect.signature(scheduler.step).parameters.keys()) + if "generator" in step_params: + kwargs["generator"] = generator + if eta is not None and "eta" in step_params: + kwargs["eta"] = eta + return kwargs + + @staticmethod + def _resolve_inference_generator( + device: Union[str, torch.device], + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + if generator is None: + return None + if isinstance(device, str): + device = torch.device(device) + device_type = device.type + + def _relocate(gen: torch.Generator) -> torch.Generator: + if gen.device.type == device_type: + return gen + return torch.Generator(device=device_type).manual_seed(gen.initial_seed()) + + if isinstance(generator, list): + return [_relocate(g) for g in generator] + return _relocate(generator) + + @staticmethod + def _prepare_generator( + generator: Optional[Union[torch.Generator, List[torch.Generator]]], + ) -> Optional[Union[torch.Generator, List[torch.Generator]]]: + """Reset generator state so repeated `__call__` with the same object is reproducible.""" + if generator is None: + return None + if isinstance(generator, list): + for gen in generator: + if gen is not None: + gen.manual_seed(int(gen.initial_seed())) + return generator + generator.manual_seed(int(generator.initial_seed())) + return generator + + def _ensure_labels_loaded(self) -> None: + if self._labels_loaded_from_model_index: + return + loaded = self._read_id2label_from_model_index(getattr(self.config, "_name_or_path", None)) + if loaded: + self._id2label = loaded + self.labels = self._build_label2id(self._id2label) + self._labels_loaded_from_model_index = True + + @staticmethod + def _normalize_id2label(id2label: Optional[Dict[Union[int, str], str]]) -> Dict[int, str]: + if not id2label: + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _read_id2label_from_model_index(variant_path: Optional[str]) -> Dict[int, str]: + if not variant_path: + return {} + model_index_path = Path(variant_path).resolve() / "model_index.json" + if not model_index_path.exists(): + return {} + raw = json.loads(model_index_path.read_text(encoding="utf-8")) + id2label = raw.get("id2label") + if not isinstance(id2label, dict): + return {} + return {int(key): value for key, value in id2label.items()} + + @staticmethod + def _build_label2id(id2label: Dict[int, str]) -> Dict[str, int]: + label2id: Dict[str, int] = {} + for class_id, value in id2label.items(): + for synonym in value.split(","): + synonym = synonym.strip() + if synonym: + label2id[synonym] = int(class_id) + return dict(sorted(label2id.items())) + + @property + def id2label(self) -> Dict[int, str]: + self._ensure_labels_loaded() + return self._id2label + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + self._ensure_labels_loaded() + if not self.labels: + raise ValueError("No labels loaded. Ensure `id2label` exists in model_index.json.") + labels = [label] if isinstance(label, str) else label + missing = [item for item in labels if item not in self.labels] + if missing: + preview = ", ".join(list(self.labels.keys())[:8]) + raise ValueError(f"Unknown label(s): {missing}. Example valid labels: {preview}, ...") + return [self.labels[item] for item in labels] + + def _normalize_class_labels(self, class_labels: Union[int, str, List[Union[int, str]]]) -> List[int]: + if isinstance(class_labels, int): + return [class_labels] + if isinstance(class_labels, str): + return self.get_label_ids(class_labels) + if class_labels and isinstance(class_labels[0], str): + return self.get_label_ids(class_labels) # type: ignore[arg-type] + return [int(class_id) for class_id in class_labels] + + def _recommended_noise_scale(self) -> float: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in RECOMMENDED_NOISE_BY_MODEL: + return RECOMMENDED_NOISE_BY_MODEL[model_type] + image_size = int(self.transformer.config.sample_size) + return {256: 1.0, 512: 2.0}.get(image_size, 1.0) + + def _default_cfg(self) -> Dict[str, float]: + model_type = getattr(self.transformer.config, "model_type", None) + if model_type in DEFAULT_CFG_BY_MODEL: + return dict(DEFAULT_CFG_BY_MODEL[model_type]) + return {"guidance_scale": 7.5, "guidance_interval_min": 0.1, "guidance_interval_max": 0.8} + + def predict_u( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + guidance_interval_min: torch.Tensor, + guidance_interval_max: torch.Tensor, + ) -> torch.Tensor: + output = self.transformer( + sample=sample, + timestep=timestep, + class_labels=class_labels, + h=h, + omega=omega, + guidance_interval_min=guidance_interval_min, + guidance_interval_max=guidance_interval_max, + return_dict=True, + ) + return output.u + + @torch.inference_mode() + def __call__( + self, + class_labels: Union[int, str, List[Union[int, str]]], + num_inference_steps: int = 1, + guidance_scale: Optional[float] = None, + guidance_interval_min: Optional[float] = None, + guidance_interval_max: Optional[float] = None, + noise_scale: Optional[float] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + if output_type not in {"pil", "np", "pt"}: + raise ValueError("output_type must be one of: 'pil', 'np', 'pt'.") + + defaults = self._default_cfg() + if guidance_scale is None: + guidance_scale = defaults["guidance_scale"] + if guidance_interval_min is None: + guidance_interval_min = defaults["guidance_interval_min"] + if guidance_interval_max is None: + guidance_interval_max = defaults["guidance_interval_max"] + if noise_scale is None: + noise_scale = self._recommended_noise_scale() + + class_label_ids = self._normalize_class_labels(class_labels) + batch_size = len(class_label_ids) + generator = self._resolve_inference_generator(self._execution_device, generator) + generator = self._prepare_generator(generator) + image_size = int(self.transformer.config.sample_size) + channels = int(self.transformer.config.in_channels) + null_class_val = int( + getattr(self.transformer.config, "num_classes", getattr(self.transformer.config, "num_class_embeds", 1000)) + ) + + latents = randn_tensor( + shape=(batch_size, channels, image_size, image_size), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) * noise_scale + + class_labels_t = torch.tensor(class_label_ids, device=self._execution_device, dtype=torch.long).reshape(-1) + class_labels_t = class_labels_t.clamp(0, null_class_val - 1) + + device = latents.device + dtype = latents.dtype + omega = torch.full((batch_size,), guidance_scale, device=device, dtype=dtype) + t_min = torch.full((batch_size,), guidance_interval_min, device=device, dtype=dtype) + t_max = torch.full((batch_size,), guidance_interval_max, device=device, dtype=dtype) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + extra_step_kwargs = self.prepare_extra_step_kwargs(self.scheduler, generator=generator) + timesteps = self.scheduler.timesteps + + for step_index in self.progress_bar(range(num_inference_steps)): + t = timesteps[step_index] + t_next = timesteps[step_index + 1] + h = (t - t_next).expand(batch_size).to(device=device, dtype=dtype) + t_batch = t.expand(batch_size).to(device=device, dtype=dtype) + + u = self.predict_u( + sample=latents, + timestep=t_batch, + class_labels=class_labels_t, + h=h, + omega=omega, + guidance_interval_min=t_min, + guidance_interval_max=t_max, + ) + latents = self.scheduler.step(u, t, latents, **extra_step_kwargs).prev_sample + + images_pt = ((latents.float().clamp(-1, 1) + 1.0) / 2.0).cpu() + if output_type == "pt": + images = images_pt + elif output_type == "np": + images = images_pt.permute(0, 2, 3, 1).numpy() + else: + images = self.numpy_to_pil(images_pt.permute(0, 2, 3, 1).numpy()) + + self.maybe_free_model_hooks() + + if not return_dict: + return (images,) + return ImagePipelineOutput(images=images) + + +PMFPipelineOutput = ImagePipelineOutput + +__all__ = ["PMFPipeline", "PMFPipelineOutput"] diff --git a/pMF-L-32-SIM/scheduler/scheduler_config.json b/pMF-L-32-SIM/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..341ff049a80407b660e0c785f274b39768b8ebce --- /dev/null +++ b/pMF-L-32-SIM/scheduler/scheduler_config.json @@ -0,0 +1,6 @@ +{ + "_class_name": "PMFScheduler", + "_diffusers_version": "0.36.0", + "num_train_timesteps": 1000, + "stochastic_sampling": false +} diff --git a/pMF-L-32-SIM/scheduler/scheduling_pmf.py b/pMF-L-32-SIM/scheduler/scheduling_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..e4a599c65bb4c64bb5296ab66e5495ac0d936139 --- /dev/null +++ b/pMF-L-32-SIM/scheduler/scheduling_pmf.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor + + +@dataclass +class PMFSchedulerOutput(BaseOutput): + prev_sample: torch.Tensor + + +class PMFScheduler(SchedulerMixin, ConfigMixin): + """Scheduler for Pixel Mean Flow sampling (t: 1 -> 0).""" + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + stochastic_sampling: bool = False, + ): + del num_train_timesteps + self.timesteps: Optional[torch.Tensor] = None + self.num_inference_steps: Optional[int] = None + self._step_index: Optional[int] = None + + @property + def init_noise_sigma(self) -> float: + return 1.0 + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device, None] = None) -> None: + if num_inference_steps < 1: + raise ValueError("num_inference_steps must be >= 1.") + self.num_inference_steps = num_inference_steps + self.timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device, dtype=torch.float32) + self._step_index = 0 + + def scale_model_input(self, sample: torch.Tensor, timestep: Union[float, torch.Tensor]) -> torch.Tensor: + del timestep + return sample + + def _resolve_step_index(self, timestep: Union[float, torch.Tensor, None]) -> int: + if self._step_index is not None: + return self._step_index + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + if timestep is None: + return 0 + + t_value = float(timestep) if not isinstance(timestep, torch.Tensor) else float(timestep.flatten()[0]) + matches = (self.timesteps - t_value).abs() < 1e-6 + if matches.any(): + return int(matches.nonzero(as_tuple=False)[0].item()) + return 0 + + @staticmethod + def _expand_time(value: torch.Tensor, sample: torch.Tensor) -> torch.Tensor: + value = value.to(dtype=sample.dtype, device=sample.device) + while value.ndim < sample.ndim: + value = value.unsqueeze(-1) + return value + + def _sample_noise( + self, + sample: torch.Tensor, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> torch.Tensor: + return randn_tensor( + sample.shape, + generator=generator, + device=sample.device, + dtype=sample.dtype, + ) + + def step( + self, + model_output: torch.Tensor, + timestep: Union[float, torch.Tensor, None], + sample: torch.Tensor, + return_dict: bool = True, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Union[PMFSchedulerOutput, Tuple[torch.Tensor]]: + if self.timesteps is None: + raise ValueError("Call `set_timesteps` before `step`.") + + step_index = self._resolve_step_index(timestep) + if step_index >= len(self.timesteps) - 1: + raise ValueError("Scheduler has already reached the final timestep.") + + t = self.timesteps[step_index] + t_next = self.timesteps[step_index + 1] + dt = self._expand_time(t - t_next, sample) + + if self.config.stochastic_sampling: + t_expanded = self._expand_time(t, sample) + t_next_expanded = self._expand_time(t_next, sample) + x0 = sample - t_expanded * model_output + noise = self._sample_noise(sample, generator=generator) + prev_sample = (1.0 - t_next_expanded) * x0 + t_next_expanded * noise + else: + prev_sample = sample - dt * model_output + + self._step_index = step_index + 1 + + if not return_dict: + return (prev_sample,) + return PMFSchedulerOutput(prev_sample=prev_sample) diff --git a/pMF-L-32-SIM/transformer/config.json b/pMF-L-32-SIM/transformer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..60f1a3453a5fe1df857b4ab24665cc9e079bd11c --- /dev/null +++ b/pMF-L-32-SIM/transformer/config.json @@ -0,0 +1,26 @@ +{ + "_class_name": "PMFTransformer2DModel", + "_diffusers_version": "0.38.0", + "aux_head_depth": 8, + "bottleneck_dim": 128, + "depth": 32, + "embedding_init_constant": 1.0, + "eval_mode": true, + "hidden_size": 1024, + "in_channels": 3, + "mlp_ratio": 2.6666666666666665, + "model_type": "pMF-L/32", + "norm_eps": 1e-06, + "num_attention_heads": 16, + "num_cfg_tokens": 4, + "num_class_embeds": null, + "num_class_tokens": 8, + "num_classes": 1000, + "num_interval_tokens": 2, + "num_time_tokens": 4, + "patch_size": 32, + "sample_size": 512, + "t_clip_min": 0.05, + "token_init_constant": 1.0, + "weight_init_constant": 0.32 +} diff --git a/pMF-L-32-SIM/transformer/diffusion_pytorch_model.safetensors b/pMF-L-32-SIM/transformer/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..992126696fdc2a02def11b3f9f7ba772bc3e92a1 --- /dev/null +++ b/pMF-L-32-SIM/transformer/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b9d18c4dcc05792b2e80ea125d96cfcb09e8a7840a687c5e6a97289078342d3 +size 1651955208 diff --git a/pMF-L-32-SIM/transformer/transformer_pmf.py b/pMF-L-32-SIM/transformer/transformer_pmf.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f97067569ec0a4d038069edca73b9c1d003596 --- /dev/null +++ b/pMF-L-32-SIM/transformer/transformer_pmf.py @@ -0,0 +1,665 @@ +from __future__ import annotations + +import math +from dataclasses import dataclass +from math import sqrt +from typing import Dict, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm +from diffusers.utils import BaseOutput + + +PMF_PRESET_CONFIGS: Dict[str, Dict[str, object]] = { + "pMF-B/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-B/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 768, + "depth": 16, + "num_attention_heads": 12, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-L/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1024, + "depth": 32, + "num_attention_heads": 16, + "bottleneck_dim": 128, + "aux_head_depth": 8, + }, + "pMF-H/16": { + "sample_size": 256, + "patch_size": 16, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, + "pMF-H/32": { + "sample_size": 512, + "patch_size": 32, + "hidden_size": 1280, + "depth": 48, + "num_attention_heads": 16, + "bottleneck_dim": 256, + "aux_head_depth": 8, + }, +} + +RECOMMENDED_NOISE_BY_MODEL: Dict[str, float] = { + "pMF-B/16": 1.0, + "pMF-B/32": 2.0, + "pMF-L/16": 1.0, + "pMF-L/32": 4.0, + "pMF-H/16": 2.0, + "pMF-H/32": 4.0, +} + +# Legacy torch repo keys (pmfDiT_*) +LEGACY_MODEL_ALIASES: Dict[str, str] = { + "pmfDiT_B_16": "pMF-B/16", + "pmfDiT_B_32": "pMF-B/32", + "pmfDiT_L_16": "pMF-L/16", + "pmfDiT_L_32": "pMF-L/32", + "pmfDiT_H_16": "pMF-H/16", + "pmfDiT_H_32": "pMF-H/32", +} + + +@dataclass +class PMFTransformer2DOutput(BaseOutput): + u: torch.Tensor + v: Optional[torch.Tensor] = None + + +def remap_legacy_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Map wrapper/backbone keys from legacy checkpoints to native PMFTransformer2DModel keys.""" + remapped: Dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + new_key = key + for prefix in ("transformer.", "net."): + if new_key.startswith(prefix): + new_key = new_key[len(prefix) :] + break + # Official PyTorch checkpoints use TorchLinear/TorchEmbedding wrappers. + new_key = new_key.replace("._flax_linear", "").replace("._flax_embedding", "") + if new_key == "rope_freqs": + continue + remapped[new_key] = value + return remapped + + +def config_from_legacy(config: Dict[str, object]) -> Dict[str, object]: + """Build native config kwargs from a legacy config.json dict.""" + model_type = config.get("model_type") or config.get("model_name") or config.get("model_str") + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type not in PMF_PRESET_CONFIGS: + raise ValueError(f"Unknown pMF preset '{model_type}'. Known: {list(PMF_PRESET_CONFIGS)}") + + preset = dict(PMF_PRESET_CONFIGS[model_type]) + preset["num_classes"] = int(config.get("num_class_embeds") or config.get("num_classes") or 1000) + preset["model_type"] = model_type + if config.get("sample_size") is not None: + preset["sample_size"] = int(config["sample_size"]) + if config.get("eval_mode") is not None: + preset["eval_mode"] = bool(config["eval_mode"]) + return preset + + +def _scaled_linear( + in_features: int, + out_features: int, + *, + bias: bool = True, + weight_init: str = "scaled_variance", + init_constant: float = 1.0, + bias_init: str = "zeros", +) -> nn.Linear: + layer = nn.Linear(in_features, out_features, bias=bias) + if weight_init == "scaled_variance": + std = init_constant / sqrt(in_features) + nn.init.normal_(layer.weight, std=std) + elif weight_init == "zeros": + nn.init.zeros_(layer.weight) + else: + raise ValueError(f"Invalid weight_init: {weight_init}") + + if bias: + if bias_init == "zeros": + nn.init.zeros_(layer.bias) + else: + raise ValueError(f"Invalid bias_init: {bias_init}") + return layer + + +class PMFTimestepEmbedder(nn.Module): + def __init__( + self, + hidden_size: int, + frequency_embedding_size: int = 256, + init_constant: float = 1.0, + ): + super().__init__() + init_kwargs = dict( + out_features=hidden_size, + bias=True, + weight_init="scaled_variance", + init_constant=init_constant, + bias_init="zeros", + ) + self.mlp = nn.Sequential( + _scaled_linear(frequency_embedding_size, **init_kwargs), + nn.SiLU(), + _scaled_linear(hidden_size, **init_kwargs), + ) + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t: torch.Tensor, dim: int, max_period: int = 10000) -> torch.Tensor: + half = dim // 2 + compute_dtype = t.dtype + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=compute_dtype, device=t.device) + / half + ) + args = t[:, None] * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t: torch.Tensor) -> torch.Tensor: + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + return self.mlp(t_freq) + + +class PMFLabelEmbedder(nn.Module): + def __init__(self, num_classes: int, hidden_size: int, init_constant: float = 1.0): + super().__init__() + self.embedding_table = nn.Embedding(num_classes + 1, hidden_size) + nn.init.normal_(self.embedding_table.weight, std=init_constant / sqrt(hidden_size)) + + def forward(self, labels: torch.Tensor) -> torch.Tensor: + return self.embedding_table(labels) + + +class PMFBottleneckPatchEmbedder(nn.Module): + def __init__( + self, + input_size: int, + patch_size: int, + pca_channels: int, + in_channels: int, + hidden_size: int, + bias: bool = True, + ): + super().__init__() + self.patch_size = (patch_size, patch_size) + self.num_patches = (input_size // patch_size) ** 2 + self.proj1 = nn.Conv2d( + in_channels, + pca_channels, + kernel_size=patch_size, + stride=patch_size, + bias=bias, + ) + self.proj2 = nn.Conv2d(pca_channels, hidden_size, kernel_size=1, stride=1, bias=bias) + + kh = kw = patch_size + fan_in = kh * kw * in_channels + fan_out = pca_channels + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj1.weight, -limit, limit) + fan_in = pca_channels + fan_out = hidden_size + limit = math.sqrt(6.0 / (fan_in + fan_out)) + nn.init.uniform_(self.proj2.weight, -limit, limit) + if bias: + nn.init.zeros_(self.proj1.bias) + nn.init.zeros_(self.proj2.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj2(self.proj1(x)) + return x.flatten(2).transpose(1, 2) + + +def precompute_rope_freqs(dim: int, seq_len: int, theta: float = 10000.0) -> torch.Tensor: + dim = dim // 2 + grid_size = int(seq_len**0.5) + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + positions = torch.arange(grid_size, dtype=torch.float32) + freqs_h = torch.einsum("i,j->ij", positions, freqs) + freqs_w = torch.einsum("i,j->ij", positions, freqs) + freqs_2d = torch.cat( + [ + torch.tile(freqs_h[:, None, :], (1, grid_size, 1)), + torch.tile(freqs_w[None, :, :], (grid_size, 1, 1)), + ], + dim=-1, + ) + real = torch.cos(freqs_2d).reshape(seq_len, dim) + imag = torch.sin(freqs_2d).reshape(seq_len, dim) + return torch.complex(real, imag) + + +def apply_rotary_pos_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + x_float = x.to(torch.float32) + x_complex = torch.view_as_complex(x_float.reshape(*x_float.shape[:-1], -1, 2).contiguous()) + freqs_cis = freqs_cis.unsqueeze(0).unsqueeze(2) + token_count = freqs_cis.shape[1] + x_rotated = x_complex.clone() + x_rotated[:, -token_count:, :] = x_complex[:, -token_count:, :] * freqs_cis + x_out = torch.view_as_real(x_rotated).flatten(-2) + return x_out.to(x.dtype) + + +class PMFAttention(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + init_kwargs = dict( + bias=False, + weight_init="scaled_variance", + init_constant=weight_init_constant, + ) + self.q_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.k_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.v_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.out_proj = _scaled_linear(hidden_size, hidden_size, **init_kwargs) + self.q_norm = RMSNorm(self.head_dim, eps=eps) + self.k_norm = RMSNorm(self.head_dim, eps=eps) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + batch_size, seq_len, channels = x.shape + q = self.q_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + k = self.k_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + v = self.v_proj(x).reshape(batch_size, seq_len, self.num_heads, self.head_dim) + + q = self.q_norm(q) + k = self.k_norm(k) + q = apply_rotary_pos_emb(q, rope_freqs) + k = apply_rotary_pos_emb(k, rope_freqs) + + query = q / math.sqrt(self.head_dim) + attn_weights = torch.einsum("bqhd,bkhd->bhqk", query, k) + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(v.dtype) + attn = torch.einsum("bhqk,bkhd->bqhd", attn_weights, v) + attn = attn.reshape(batch_size, seq_len, channels) + return self.out_proj(attn) + + +class PMFSwiGLUMlp(nn.Module): + def __init__(self, dim: int, hidden_dim: int, weight_init_constant: float = 0.32): + super().__init__() + init_kwargs = dict(bias=False, weight_init="scaled_variance", init_constant=weight_init_constant) + self.w1 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w3 = _scaled_linear(dim, hidden_dim, **init_kwargs) + self.w2 = _scaled_linear(hidden_dim, dim, **init_kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.w2(F.silu(self.w1(x)) * self.w3(x)) + + +class PMFTransformerBlock(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float = 8 / 3, + weight_init_constant: float = 0.32, + eps: float = 1e-6, + ): + super().__init__() + self.norm1 = RMSNorm(hidden_size, eps=eps) + self.attn = PMFAttention(hidden_size, num_heads, weight_init_constant=weight_init_constant, eps=eps) + self.norm2 = RMSNorm(hidden_size, eps=eps) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + if hidden_size > 1024: + mlp_hidden_dim = (mlp_hidden_dim + 7) // 8 * 8 + self.mlp = PMFSwiGLUMlp(hidden_size, mlp_hidden_dim, weight_init_constant=weight_init_constant) + self.attn_scale = nn.Parameter(torch.zeros(hidden_size)) + self.mlp_scale = nn.Parameter(torch.zeros(hidden_size)) + + def forward(self, x: torch.Tensor, rope_freqs: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), rope_freqs) * self.attn_scale + x = x + self.mlp(self.norm2(x)) * self.mlp_scale + return x + + +class PMFFinalLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int, eps: float = 1e-6): + super().__init__() + self.norm = RMSNorm(hidden_size, eps=eps) + self.linear = _scaled_linear( + hidden_size, + patch_size * patch_size * out_channels, + bias=True, + weight_init="zeros", + bias_init="zeros", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(self.norm(x)) + + +class PMFTransformer2DModel(ModelMixin, ConfigMixin): + """Native diffusers implementation of the pMF DiT backbone.""" + + _supports_gradient_checkpointing = True + _skip_layerwise_casting_patterns = ["pos_embed", "rope_freqs"] + + @register_to_config + def __init__( + self, + sample_size: int = 256, + patch_size: int = 16, + in_channels: int = 3, + hidden_size: int = 768, + depth: int = 16, + num_attention_heads: int = 12, + mlp_ratio: float = 8 / 3, + num_classes: int = 1000, + bottleneck_dim: int = 128, + aux_head_depth: int = 8, + num_class_tokens: int = 8, + num_time_tokens: int = 4, + num_cfg_tokens: int = 4, + num_interval_tokens: int = 2, + token_init_constant: float = 1.0, + embedding_init_constant: float = 1.0, + weight_init_constant: float = 0.32, + eval_mode: bool = True, + model_type: str | None = None, + num_class_embeds: int | None = None, + t_clip_min: float = 0.05, + norm_eps: float = 1e-6, + ): + super().__init__() + if num_class_embeds is not None: + num_classes = int(num_class_embeds) + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type in PMF_PRESET_CONFIGS: + preset = PMF_PRESET_CONFIGS[model_type] + sample_size = int(preset["sample_size"]) + patch_size = int(preset["patch_size"]) + hidden_size = int(preset["hidden_size"]) + depth = int(preset["depth"]) + num_attention_heads = int(preset["num_attention_heads"]) + bottleneck_dim = int(preset["bottleneck_dim"]) + aux_head_depth = int(preset["aux_head_depth"]) + + self.sample_size = sample_size + self.patch_size = patch_size + self.in_channels = in_channels + self.out_channels = in_channels + self.hidden_size = hidden_size + self.depth = depth + self.num_attention_heads = num_attention_heads + self.aux_head_depth = aux_head_depth + self.num_class_tokens = num_class_tokens + self.num_time_tokens = num_time_tokens + self.num_cfg_tokens = num_cfg_tokens + self.num_interval_tokens = num_interval_tokens + self.prefix_tokens = ( + num_class_tokens + num_cfg_tokens + 2 * num_interval_tokens + num_time_tokens + ) + self.t_clip_min = t_clip_min + self.eval_mode = eval_mode + self.gradient_checkpointing = False + + self.x_embedder = PMFBottleneckPatchEmbedder( + sample_size, + patch_size, + bottleneck_dim, + in_channels, + hidden_size, + bias=True, + ) + embed_kwargs = dict(hidden_size=hidden_size, init_constant=embedding_init_constant) + self.h_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.omega_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_start_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.cfg_t_end_embedder = PMFTimestepEmbedder(**embed_kwargs) + self.y_embedder = PMFLabelEmbedder(num_classes, hidden_size, init_constant=embedding_init_constant) + + token_std = token_init_constant / math.sqrt(hidden_size) + self.time_tokens = nn.Parameter(torch.randn(1, num_time_tokens, hidden_size) * token_std) + self.class_tokens = nn.Parameter(torch.randn(1, num_class_tokens, hidden_size) * token_std) + self.omega_tokens = nn.Parameter(torch.randn(1, num_cfg_tokens, hidden_size) * token_std) + self.t_min_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + self.t_max_tokens = nn.Parameter(torch.randn(1, num_interval_tokens, hidden_size) * token_std) + + total_tokens = self.x_embedder.num_patches + self.prefix_tokens + self.pos_embed = nn.Parameter(torch.randn(1, total_tokens, hidden_size) * 0.02) + + head_dim = hidden_size // num_attention_heads + self.register_buffer( + "rope_freqs", + precompute_rope_freqs(head_dim, self.x_embedder.num_patches), + persistent=False, + ) + + shared_depth = depth - aux_head_depth + block_kwargs = dict( + hidden_size=hidden_size, + num_heads=num_attention_heads, + mlp_ratio=mlp_ratio, + weight_init_constant=weight_init_constant, + eps=norm_eps, + ) + self.shared_blocks = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(shared_depth)]) + self.u_heads = nn.ModuleList([PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth)]) + self.v_heads = nn.ModuleList( + [PMFTransformerBlock(**block_kwargs) for _ in range(aux_head_depth if not eval_mode else 0)] + ) + self.u_final_layer = PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + self.v_final_layer = ( + PMFFinalLayer(hidden_size, patch_size, in_channels, eps=norm_eps) + if not eval_mode + else None + ) + + def _build_sequence( + self, + sample: torch.Tensor, + h: torch.Tensor, + omega: torch.Tensor, + t_min: torch.Tensor, + t_max: torch.Tensor, + class_labels: torch.Tensor, + ) -> torch.Tensor: + x_embed = self.x_embedder(sample) + h_embed = self.h_embedder(h) + omega_embed = self.omega_embedder(1 - 1 / omega) + t_min_embed = self.cfg_t_start_embedder(t_min) + t_max_embed = self.cfg_t_end_embedder(t_max) + y_embed = self.y_embedder(class_labels) + + time_tokens = self.time_tokens + h_embed.unsqueeze(1) + omega_tokens = self.omega_tokens + omega_embed.unsqueeze(1) + t_min_tokens = self.t_min_tokens + t_min_embed.unsqueeze(1) + t_max_tokens = self.t_max_tokens + t_max_embed.unsqueeze(1) + class_tokens = self.class_tokens + y_embed.unsqueeze(1) + + seq = torch.cat( + [class_tokens, omega_tokens, t_min_tokens, t_max_tokens, time_tokens, x_embed], + dim=1, + ) + return seq + self.pos_embed + + def _unpatchify(self, tokens: torch.Tensor) -> torch.Tensor: + batch_size = tokens.shape[0] + patch = self.patch_size + grid = int(tokens.shape[1] ** 0.5) + channels = self.out_channels + x = tokens.reshape(batch_size, grid, grid, patch, patch, channels) + x = torch.einsum("nhwpqc->nchpwq", x) + return x.reshape(batch_size, channels, grid * patch, grid * patch) + + def forward( + self, + sample: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.Tensor, + h: Optional[torch.Tensor] = None, + omega: Optional[torch.Tensor] = None, + guidance_interval_min: Optional[torch.Tensor] = None, + guidance_interval_max: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> PMFTransformer2DOutput | Tuple[torch.Tensor, Optional[torch.Tensor]]: + batch_size = sample.shape[0] + timestep = self._expand_batch(timestep, batch_size, sample.device, sample.dtype) + h = self._expand_batch(h if h is not None else timestep, batch_size, sample.device, sample.dtype) + omega = self._expand_batch( + omega if omega is not None else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_min = self._expand_batch( + guidance_interval_min + if guidance_interval_min is not None + else torch.zeros(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + guidance_interval_max = self._expand_batch( + guidance_interval_max + if guidance_interval_max is not None + else torch.ones(batch_size, device=sample.device), + batch_size, + sample.device, + sample.dtype, + ) + + seq = self._build_sequence(sample, h, omega, guidance_interval_min, guidance_interval_max, class_labels) + rope_freqs = self.rope_freqs.to(device=sample.device) + + for block in self.shared_blocks: + if self.training and self.gradient_checkpointing: + seq = torch.utils.checkpoint.checkpoint(block, seq, rope_freqs, use_reentrant=False) + else: + seq = block(seq, rope_freqs) + + u_seq = v_seq = seq + for block in self.u_heads: + if self.training and self.gradient_checkpointing: + u_seq = torch.utils.checkpoint.checkpoint(block, u_seq, rope_freqs, use_reentrant=False) + else: + u_seq = block(u_seq, rope_freqs) + + for block in self.v_heads: + if self.training and self.gradient_checkpointing: + v_seq = torch.utils.checkpoint.checkpoint(block, v_seq, rope_freqs, use_reentrant=False) + else: + v_seq = block(v_seq, rope_freqs) + + u_tokens = u_seq[:, self.prefix_tokens :] + u_pred = self._unpatchify(self.u_final_layer(u_tokens)) + t = timestep.reshape(batch_size, 1, 1, 1) + u = (sample - u_pred) / torch.clamp(t, min=self.t_clip_min) + + v = None + if self.v_final_layer is not None: + v_tokens = v_seq[:, self.prefix_tokens :] + v_pred = self._unpatchify(self.v_final_layer(v_tokens)) + v = (sample - v_pred) / torch.clamp(t, min=self.t_clip_min) + + if not return_dict: + return (u, v) + return PMFTransformer2DOutput(u=u, v=v) + + @staticmethod + def _expand_batch( + value: torch.Tensor, + batch_size: int, + device: torch.device, + dtype: torch.dtype, + ) -> torch.Tensor: + value = torch.as_tensor(value, device=device, dtype=dtype) + if value.ndim == 0: + value = value.reshape(1) + if value.shape[0] == 1 and batch_size > 1: + value = value.expand(batch_size) + return value.reshape(batch_size) + + @classmethod + def from_pmf_checkpoint( + cls, + checkpoint_path: str, + model_type: str | None = None, + map_location: str = "cpu", + strict: bool = False, + ) -> Tuple["PMFTransformer2DModel", Dict[str, object]]: + checkpoint = torch.load(checkpoint_path, map_location=map_location, weights_only=False) + if isinstance(checkpoint, dict) and "state_dict" in checkpoint: + state_dict = checkpoint["state_dict"] + else: + state_dict = checkpoint + + if model_type is None: + for key in ("model_type", "model_str", "model"): + if isinstance(checkpoint, dict) and key in checkpoint: + model_type = checkpoint[key] + break + if model_type in LEGACY_MODEL_ALIASES: + model_type = LEGACY_MODEL_ALIASES[model_type] + if model_type is None: + raise ValueError("model_type is required when it cannot be inferred from the checkpoint.") + + config = dict(PMF_PRESET_CONFIGS[model_type]) + config["model_type"] = model_type + config["eval_mode"] = True + model = cls(**config) + model.load_state_dict(remap_legacy_state_dict(state_dict), strict=strict) + metadata = {"checkpoint_path": checkpoint_path, "model_type": model_type} + return model, metadata + + def to_pmf_checkpoint(self, prefix: str = "net.") -> Dict[str, torch.Tensor]: + state_dict: Dict[str, torch.Tensor] = {} + for key, value in self.state_dict().items(): + state_dict[f"{prefix}{key}"] = value.detach().cpu() + return state_dict + + @property + def net(self): + return self + + +PMFDiffusersModel = PMFTransformer2DModel