File size: 6,277 Bytes
1435032 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | {
"disclaimer_en": "Informal community evaluation for reference only; not an official evaluation.",
"disclaimer_zh": "社区非正式测评，仅供参考，不代表任何官方评价。",
"cases": [
{
"id": "sweater_red",
"en": "Older woman: blue sweater to red",
"zh": "老年女性：蓝色毛衣改为红色",
"prompt": "Change only the blue sweater to a red sweater. Preserve the same person, face, pose, lighting and background.",
"input": "evaluation/audit/input.png",
"bf16": "evaluation/audit/bf16-native-tiled/edit_native.png",
"int8": "evaluation/audit/int8-native-tiled/edit_native.png",
"observation_en": "Both outputs make the sweater red and retain soft lighting and broadly natural skin texture. Fine facial and fabric details still shift, and faint colored marks are visible in both; this is not lossless preservation.",
"observation_zh": "两组均将毛衣改为红色，较好保留柔和光线与自然皮肤质感；面部和织物细节仍有偏移，两组均可见细小彩色痕迹，并非无损保持。",
"input_sha256": "01e37e178fb0d35d5c706b7edd76bc1a6c5b50ec3536901a467fd45295d097aa",
"bf16_sha256": "c551ac809ccb48146d1ad842d75975c0e9663d388a368010d2bfccb7883eeeec",
"int8_sha256": "b6af18d8688bb2669632adf0cfae962bc34431e30e935014997010cde5c2ca59"
},
{
"id": "portrait_recolor",
"en": "Adult woman: cream sweater to navy blue",
"zh": "成年女性：米白毛衣改为藏蓝",
"prompt": "Change only the cream sweater to a navy blue sweater. Preserve the same woman, face, hairstyle, pose, lighting and background.",
"input": "evaluation/editing/inputs/portrait_recolor.png",
"bf16": "evaluation/editing/bf16/portrait_recolor-s42.png",
"int8": "evaluation/editing/int8/portrait_recolor-s42.png",
"observation_en": "Both outputs recolor the sweater to navy blue while broadly retaining the woman, pose, window light and background. Facial, hair and knitted-fabric details still shift slightly; this is not an identity-preservation score.",
"observation_zh": "两组均将毛衣改为藏蓝色，大体保持人物、姿态、窗边光线与背景。五官、发丝和针织纹理仍有轻微变化，这不是身份保持评分。",
"input_sha256": "00716ad4721f585be4b0bf0daf2413f4ab92b795f1eb4d08b4f1ecc604b075b8",
"bf16_sha256": "a215c639d838aa9080346a6701008ce991e90f278db175ec7f110bf244f19e7c",
"int8_sha256": "d9d566f33d7d8c82b65b009766212d2b68d0f34e1b9a42e538af979b59c1bda4"
},
{
"id": "cup_material",
"en": "Product edit: ceramic cup to brushed metal",
"zh": "物品编辑：陶瓷杯改为拉丝金属",
"prompt": "Change only the middle blue ceramic cup into a brushed silver metal cup. Preserve its shape and position, the red and yellow cups, the green apple, the tabletop, lighting and background.",
"input": "evaluation/editing/inputs/cup_material.png",
"bf16": "evaluation/editing/bf16/cup_material-s42.png",
"int8": "evaluation/editing/int8/cup_material-s42.png",
"observation_en": "Both make the middle cup look like brushed silver metal while broadly preserving the arrangement of three cups and the apple. Cup rims, reflections and apple texture shift slightly; faint line artifacts are visible in the background.",
"observation_zh": "两组均将中间杯子呈现为银色拉丝金属外观，大体保持三杯与苹果的排列。杯口、反光和苹果纹理有轻微变化，背景可见细线状痕迹。",
"input_sha256": "91e967685fbbd79ba13daece22a0f72c863cb518db88d5d2a6f9aecfc5f292b1",
"bf16_sha256": "4c5f4f22843ca58ff8f52ca5f5b62061083b746dc3f14cb0a1827bcfcdc9df6e",
"int8_sha256": "9177ea912de8d1c6e7c99782d939acbaaa0049da14784b4b5fdb81b4390d4ce0"
},
{
"id": "bird_background",
"en": "Background replacement: snowy forest (unsuccessful)",
"zh": "背景替换：雪林（未达到目标）",
"prompt": "Replace only the blurred forest background with a softly blurred snowy winter forest. Preserve the same kingfisher, its colors and pose, the mossy branch, framing and foreground lighting.",
"input": "evaluation/editing/inputs/bird_background.png",
"bf16": "evaluation/editing/bf16/bird_background-s42.png",
"int8": "evaluation/editing/int8/bird_background-s42.png",
"observation_en": "Both fail the requested edit: the mossy branch disappears, the background becomes largely white instead of a snowy forest, and the bird loses photographic detail. This failure is present in both BF16 and INT8 and cannot be attributed to quantization alone.",
"observation_zh": "两组均未达到编辑目标：苔藓枝干消失，背景大面积变白而非雪林，鸟的摄影细节也明显损失。BF16 与 INT8 均存在此失败，不能将其单独归因于量化。",
"input_sha256": "40a0f73bb407a394fa9e8a3f6922455df716b9725ad07437fa10c1daca2c2095",
"bf16_sha256": "040480b513d46eadb0d13910845d1f539fad7152dd57375892bb574969430ef8",
"int8_sha256": "7afde64515861b1312388f88edf94c25dc146b388107cd223befafc3e22e3eac"
}
],
"measurements": [
{
"case_id": "portrait_recolor",
"bf16_seconds": 201.80096119999507,
"int8_seconds": 194.7663809000005,
"bf16_peak_gib": 26.51371479034424,
"int8_peak_gib": 21.646138191223145
},
{
"case_id": "cup_material",
"bf16_seconds": 433.98883840000053,
"int8_seconds": 193.33272730000317,
"bf16_peak_gib": 26.52234125137329,
"int8_peak_gib": 21.656850337982178
},
{
"case_id": "bird_background",
"bf16_seconds": 585.3128182,
"int8_seconds": 184.6775536999994,
"bf16_peak_gib": 26.521997451782227,
"int8_peak_gib": 21.65667486190796
}
],
"case_file_note": "Published JSON text may use LF line endings. measured-cases.zip preserves the exact editing.json bytes whose SHA256 appears in both environment records; parsed case definitions are identical.",
"timing_note": "Three declared cases, one excluded seed-0 warmup per precision, one measured seed-42 call per case. The separate older-woman cold instrumented call is excluded."
} |