| [ |
| { |
| "checkpoint": "checkpoint-21350", |
| "id_acc": 0.9110954861111111, |
| "ood_acc": 0.35171796875, |
| "total_acc": 0.6166862664473685, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 199.92764930555555, |
| "ood_avg_resp_len": 320.44338671875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 499909, |
| "answer_accuracy": 0.617378922304774, |
| "avg_response_len": 251.66101456291494, |
| "resp_tokens_sum": 203776970, |
| "pass_at_k": { |
| "pass@1": 0.617378922304774, |
| "pass@2": 0.6702401358535359, |
| "pass@4": 0.7126881271914993, |
| "pass@8": 0.7469478039476964, |
| "pass@16": 0.7751029216952456, |
| "pass@32": 0.7984062550092523, |
| "pass@64": 0.8175423265721407, |
| "pass@128": 0.833386025924755 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8190680309734514, |
| "pass@2": 0.8758038783592319, |
| "pass@4": 0.9097118844657693, |
| "pass@8": 0.9301279923292359, |
| "pass@16": 0.945463319652103, |
| "pass@32": 0.9597276818955355, |
| "pass@64": 0.971984946225041, |
| "pass@128": 0.9793510324483776 |
| }, |
| "2": { |
| "pass@1": 0.9998539719626168, |
| "pass@2": 0.9999996167243114, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9974197247706422, |
| "pass@2": 0.9979062071082859, |
| "pass@4": 0.9985759549551719, |
| "pass@8": 0.9993549275868031, |
| "pass@16": 0.9998791891503424, |
| "pass@32": 0.999997274769305, |
| "pass@64": 0.999999999917904, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9796875, |
| "pass@2": 0.9833226962352362, |
| "pass@4": 0.9855182915612108, |
| "pass@8": 0.9877684577554199, |
| "pass@16": 0.9898908624050529, |
| "pass@32": 0.9915912787745874, |
| "pass@64": 0.9929543613492345, |
| "pass@128": 0.99375 |
| }, |
| "5": { |
| "pass@1": 0.8453621031746031, |
| "pass@2": 0.878759686289214, |
| "pass@4": 0.9038273906237906, |
| "pass@8": 0.9236315572169654, |
| "pass@16": 0.9401206361112525, |
| "pass@32": 0.9527244490497119, |
| "pass@64": 0.9623853430676196, |
| "pass@128": 0.9682539682539683 |
| }, |
| "6": { |
| "pass@1": 0.9475580089820359, |
| "pass@2": 0.9595694192324015, |
| "pass@4": 0.9684871766590554, |
| "pass@8": 0.9760311732040033, |
| "pass@16": 0.9819037036422803, |
| "pass@32": 0.9859049285559663, |
| "pass@64": 0.9889859412882993, |
| "pass@128": 0.9910179640718563 |
| }, |
| "7": { |
| "pass@1": 0.9068047337278107, |
| "pass@2": 0.932142305945115, |
| "pass@4": 0.947185020234157, |
| "pass@8": 0.9582908289110658, |
| "pass@16": 0.9681797653463373, |
| "pass@32": 0.9767144305121371, |
| "pass@64": 0.9830296516855794, |
| "pass@128": 0.9881656804733728 |
| }, |
| "8": { |
| "pass@1": 0.8812349759615384, |
| "pass@2": 0.9134710024227741, |
| "pass@4": 0.9344750253574073, |
| "pass@8": 0.9487608950939497, |
| "pass@16": 0.9589510177536014, |
| "pass@32": 0.9657383512461981, |
| "pass@64": 0.9703603198365051, |
| "pass@128": 0.9743589743589743 |
| }, |
| "9": { |
| "pass@1": 0.8417913732394366, |
| "pass@2": 0.8856545275590555, |
| "pass@4": 0.9142673059353488, |
| "pass@8": 0.9349954551526746, |
| "pass@16": 0.9509292121287449, |
| "pass@32": 0.9633821177411638, |
| "pass@64": 0.973017756199603, |
| "pass@128": 0.9802816901408451 |
| }, |
| "11": { |
| "pass@1": 0.7452195467422096, |
| "pass@2": 0.8180329320113316, |
| "pass@4": 0.8652162037641893, |
| "pass@8": 0.8969106210163632, |
| "pass@16": 0.9210920894565872, |
| "pass@32": 0.9403227057391875, |
| "pass@64": 0.9548937056898483, |
| "pass@128": 0.9660056657223796 |
| }, |
| "12": { |
| "pass@1": 0.6948490466101694, |
| "pass@2": 0.7856954685484225, |
| "pass@4": 0.8424545043628017, |
| "pass@8": 0.8799306789509234, |
| "pass@16": 0.9062520096316939, |
| "pass@32": 0.924809518611065, |
| "pass@64": 0.938702946618994, |
| "pass@128": 0.9491525423728814 |
| }, |
| "13": { |
| "pass@1": 0.5179516065830722, |
| "pass@2": 0.6325184200133289, |
| "pass@4": 0.7205692302764656, |
| "pass@8": 0.7828447072240617, |
| "pass@16": 0.8262471661892159, |
| "pass@32": 0.8598153482049776, |
| "pass@64": 0.8862313514669503, |
| "pass@128": 0.9059561128526645 |
| }, |
| "14": { |
| "pass@1": 0.3964417834890966, |
| "pass@2": 0.5047361376848922, |
| "pass@4": 0.5946405522323733, |
| "pass@8": 0.6680429180152108, |
| "pass@16": 0.7301454909544141, |
| "pass@32": 0.7804329625460306, |
| "pass@64": 0.8173909998798203, |
| "pass@128": 0.8411214953271028 |
| }, |
| "15": { |
| "pass@1": 0.29612987716763006, |
| "pass@2": 0.3869219653179187, |
| "pass@4": 0.46993725170191875, |
| "pass@8": 0.5421051675527985, |
| "pass@16": 0.6023771645269487, |
| "pass@32": 0.6529647288435331, |
| "pass@64": 0.6944795629555712, |
| "pass@128": 0.7283236994219653 |
| }, |
| "16": { |
| "pass@1": 0.24740423387096774, |
| "pass@2": 0.32904813309626596, |
| "pass@4": 0.40458356405046153, |
| "pass@8": 0.4702012137774771, |
| "pass@16": 0.5247288669662102, |
| "pass@32": 0.5706876527594704, |
| "pass@64": 0.6115942057418606, |
| "pass@128": 0.6451612903225806 |
| }, |
| "17": { |
| "pass@1": 0.16986542630057805, |
| "pass@2": 0.22985803729916698, |
| "pass@4": 0.28688736695124073, |
| "pass@8": 0.33843467932243226, |
| "pass@16": 0.38700181611143314, |
| "pass@32": 0.43410878184626833, |
| "pass@64": 0.47804135764926775, |
| "pass@128": 0.5202312138728323 |
| }, |
| "18": { |
| "pass@1": 0.1679017857142857, |
| "pass@2": 0.2257529527559055, |
| "pass@4": 0.2848023300658846, |
| "pass@8": 0.34394438573386016, |
| "pass@16": 0.3988663230247612, |
| "pass@32": 0.4435387309873241, |
| "pass@64": 0.47824795716449336, |
| "pass@128": 0.5085714285714286 |
| }, |
| "19": { |
| "pass@1": 0.16246715465465467, |
| "pass@2": 0.2220851510959779, |
| "pass@4": 0.2795086211070464, |
| "pass@8": 0.3279673225850375, |
| "pass@16": 0.36686001138513913, |
| "pass@32": 0.4010357183857945, |
| "pass@64": 0.4330668055174505, |
| "pass@128": 0.46546546546546547 |
| }, |
| "20": { |
| "pass@1": 0.15582770270270271, |
| "pass@2": 0.21083792946489788, |
| "pass@4": 0.2651404334593312, |
| "pass@8": 0.31510224505908696, |
| "pass@16": 0.3582141742580615, |
| "pass@32": 0.3942354522194376, |
| "pass@64": 0.4242069841105825, |
| "pass@128": 0.45345345345345345 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 507526, |
| "answer_accuracy": 0.6180899259547935, |
| "avg_response_len": 260.83856196414655, |
| "resp_tokens_sum": 214179760, |
| "pass_at_k": { |
| "pass@1": 0.6180899259547935, |
| "pass@2": 0.6704741094936216, |
| "pass@4": 0.714109435375526, |
| "pass@8": 0.7502174513426444, |
| "pass@16": 0.7800515230208226, |
| "pass@32": 0.804595822219313, |
| "pass@64": 0.8246596996146075, |
| "pass@128": 0.8419329696024942 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.7672785659509203, |
| "pass@2": 0.8221017402540941, |
| "pass@4": 0.8567607347586148, |
| "pass@8": 0.8803886350813609, |
| "pass@16": 0.8990823514930147, |
| "pass@32": 0.9155576005344653, |
| "pass@64": 0.9292128475444902, |
| "pass@128": 0.9386503067484663 |
| }, |
| "2": { |
| "pass@1": 0.9999762537993921, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9975378787878788, |
| "pass@2": 0.999380368647101, |
| "pass@4": 0.9999463317085363, |
| "pass@8": 0.999999574463101, |
| "pass@16": 0.99999999998634, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9803069526627219, |
| "pass@2": 0.9856821186460419, |
| "pass@4": 0.9884984804968907, |
| "pass@8": 0.9901388748274397, |
| "pass@16": 0.9915489433224962, |
| "pass@32": 0.993055431506491, |
| "pass@64": 0.9948274051186248, |
| "pass@128": 0.9970414201183432 |
| }, |
| "5": { |
| "pass@1": 0.8750452898550725, |
| "pass@2": 0.8991081108068009, |
| "pass@4": 0.91609362552507, |
| "pass@8": 0.9283145339273063, |
| "pass@16": 0.9387362873527924, |
| "pass@32": 0.9495945434273302, |
| "pass@64": 0.9611962026508659, |
| "pass@128": 0.9739130434782609 |
| }, |
| "6": { |
| "pass@1": 0.9597063037249284, |
| "pass@2": 0.9688247354646573, |
| "pass@4": 0.9755671450810769, |
| "pass@8": 0.9809583841351045, |
| "pass@16": 0.9856212779919914, |
| "pass@32": 0.9895090347929678, |
| "pass@64": 0.9923046542590954, |
| "pass@128": 0.994269340974212 |
| }, |
| "7": { |
| "pass@1": 0.9184429190751445, |
| "pass@2": 0.9380771101679485, |
| "pass@4": 0.9515284368355686, |
| "pass@8": 0.9620686520464642, |
| "pass@16": 0.9694759351290418, |
| "pass@32": 0.9729727131601149, |
| "pass@64": 0.9739292569091923, |
| "pass@128": 0.9739884393063584 |
| }, |
| "8": { |
| "pass@1": 0.8584507042253521, |
| "pass@2": 0.8973130614395034, |
| "pass@4": 0.9233392773966632, |
| "pass@8": 0.9433777973806029, |
| "pass@16": 0.9601626523567398, |
| "pass@32": 0.9730733897077097, |
| "pass@64": 0.9801915294569885, |
| "pass@128": 0.9830985915492958 |
| }, |
| "9": { |
| "pass@1": 0.8166401571856288, |
| "pass@2": 0.8604763897402046, |
| "pass@4": 0.8899476901090955, |
| "pass@8": 0.9137484797589711, |
| "pass@16": 0.9336248046731362, |
| "pass@32": 0.9489223240336991, |
| "pass@64": 0.9611196746977648, |
| "pass@128": 0.9760479041916168 |
| }, |
| "11": { |
| "pass@1": 0.759765625, |
| "pass@2": 0.8237484880466749, |
| "pass@4": 0.8661105835038693, |
| "pass@8": 0.8974093782721269, |
| "pass@16": 0.9220586483640653, |
| "pass@32": 0.9400776540050912, |
| "pass@64": 0.9516687417338571, |
| "pass@128": 0.9578313253012049 |
| }, |
| "12": { |
| "pass@1": 0.6983248432601881, |
| "pass@2": 0.792604765754202, |
| "pass@4": 0.8538179523210068, |
| "pass@8": 0.8929835881620504, |
| "pass@16": 0.9185320929903009, |
| "pass@32": 0.9352178761761952, |
| "pass@64": 0.9478804552757559, |
| "pass@128": 0.9592476489028213 |
| }, |
| "13": { |
| "pass@1": 0.5278361344537815, |
| "pass@2": 0.63536910827323, |
| "pass@4": 0.7126288210822387, |
| "pass@8": 0.7701978399669224, |
| "pass@16": 0.8166211744373886, |
| "pass@32": 0.8550839822738148, |
| "pass@64": 0.8865038689796082, |
| "pass@128": 0.9131652661064426 |
| }, |
| "14": { |
| "pass@1": 0.41721491228070173, |
| "pass@2": 0.5287199400239444, |
| "pass@4": 0.6211705283001904, |
| "pass@8": 0.6923516248529362, |
| "pass@16": 0.7462194130033619, |
| "pass@32": 0.7883736937027563, |
| "pass@64": 0.8222691206866892, |
| "pass@128": 0.8508771929824561 |
| }, |
| "15": { |
| "pass@1": 0.28927556818181815, |
| "pass@2": 0.38632150739680254, |
| "pass@4": 0.477739166411017, |
| "pass@8": 0.5566535711234109, |
| "pass@16": 0.6215893466194053, |
| "pass@32": 0.6719853954938052, |
| "pass@64": 0.7093504805012789, |
| "pass@128": 0.7393939393939394 |
| }, |
| "16": { |
| "pass@1": 0.21276264245014245, |
| "pass@2": 0.28995263813177213, |
| "pass@4": 0.3666882563824822, |
| "pass@8": 0.4346854612065274, |
| "pass@16": 0.490842220216734, |
| "pass@32": 0.5378512803345692, |
| "pass@64": 0.5810806305675018, |
| "pass@128": 0.6239316239316239 |
| }, |
| "17": { |
| "pass@1": 0.19004755434782608, |
| "pass@2": 0.2589851200665132, |
| "pass@4": 0.3332288832684735, |
| "pass@8": 0.4048084183288024, |
| "pass@16": 0.46686980557788155, |
| "pass@32": 0.5180633036985619, |
| "pass@64": 0.5611210047877057, |
| "pass@128": 0.5962732919254659 |
| }, |
| "18": { |
| "pass@1": 0.17541360294117647, |
| "pass@2": 0.238078248031496, |
| "pass@4": 0.3021544674562736, |
| "pass@8": 0.36484250871024415, |
| "pass@16": 0.42552854294852027, |
| "pass@32": 0.4830248444609159, |
| "pass@64": 0.532842036580717, |
| "pass@128": 0.5705882352941176 |
| }, |
| "19": { |
| "pass@1": 0.14276413690476192, |
| "pass@2": 0.19880974350862385, |
| "pass@4": 0.25782905819808244, |
| "pass@8": 0.3139917285710976, |
| "pass@16": 0.3642751043986496, |
| "pass@32": 0.4084618548346345, |
| "pass@64": 0.44438723572602956, |
| "pass@128": 0.47619047619047616 |
| }, |
| "20": { |
| "pass@1": 0.14429734281437126, |
| "pass@2": 0.2033662743410817, |
| "pass@4": 0.26359721810597025, |
| "pass@8": 0.3163735906613064, |
| "pass@16": 0.359637297892788, |
| "pass@32": 0.395871809493765, |
| "pass@64": 0.42783318947128957, |
| "pass@128": 0.46107784431137727 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 492346, |
| "answer_accuracy": 0.6145475515258029, |
| "avg_response_len": 277.75934778918355, |
| "resp_tokens_sum": 222527457, |
| "pass_at_k": { |
| "pass@1": 0.6145475515258029, |
| "pass@2": 0.66511725084383, |
| "pass@4": 0.7072032517044973, |
| "pass@8": 0.7420143982533723, |
| "pass@16": 0.7702823493585079, |
| "pass@32": 0.7934652060341785, |
| "pass@64": 0.8130399718603196, |
| "pass@128": 0.8304841028918357 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.837080223880597, |
| "pass@2": 0.8841825567046653, |
| "pass@4": 0.9129999748165799, |
| "pass@8": 0.9340309207425049, |
| "pass@16": 0.9503269298903634, |
| "pass@32": 0.9633252118626652, |
| "pass@64": 0.9742339636240699, |
| "pass@128": 0.9850746268656716 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9994989067055393, |
| "pass@2": 0.9999314897500058, |
| "pass@4": 0.9999986759125955, |
| "pass@8": 0.9999999997431218, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9685215643274854, |
| "pass@2": 0.9750720921397983, |
| "pass@4": 0.9805761952453314, |
| "pass@8": 0.9847048683048196, |
| "pass@16": 0.9872249490269424, |
| "pass@32": 0.988723427016145, |
| "pass@64": 0.9897569491732896, |
| "pass@128": 0.9912280701754386 |
| }, |
| "5": { |
| "pass@1": 0.8611213235294117, |
| "pass@2": 0.8884335919407136, |
| "pass@4": 0.9081887436864504, |
| "pass@8": 0.924253573036253, |
| "pass@16": 0.9389244449350933, |
| "pass@32": 0.9523447840529764, |
| "pass@64": 0.9630246066646815, |
| "pass@128": 0.9705882352941176 |
| }, |
| "6": { |
| "pass@1": 0.9733832807570978, |
| "pass@2": 0.9828349716833503, |
| "pass@4": 0.9887145172508008, |
| "pass@8": 0.9916570561272308, |
| "pass@16": 0.9930273048146369, |
| "pass@32": 0.9941869150417751, |
| "pass@64": 0.9952583726678965, |
| "pass@128": 0.9968454258675079 |
| }, |
| "7": { |
| "pass@1": 0.8899080300632911, |
| "pass@2": 0.9240950176916176, |
| "pass@4": 0.945723327898886, |
| "pass@8": 0.9597153776968665, |
| "pass@16": 0.9702119965930145, |
| "pass@32": 0.9779051128332572, |
| "pass@64": 0.9826542051427243, |
| "pass@128": 0.9841772151898734 |
| }, |
| "8": { |
| "pass@1": 0.8529466966966966, |
| "pass@2": 0.8928994200893807, |
| "pass@4": 0.9212113263432158, |
| "pass@8": 0.94219890076564, |
| "pass@16": 0.956245171195436, |
| "pass@32": 0.9650723844889141, |
| "pass@64": 0.9708403647709795, |
| "pass@128": 0.972972972972973 |
| }, |
| "9": { |
| "pass@1": 0.8221463022508039, |
| "pass@2": 0.8635371040838545, |
| "pass@4": 0.8914274667635996, |
| "pass@8": 0.9129749581292425, |
| "pass@16": 0.9306763023858616, |
| "pass@32": 0.9479524332552338, |
| "pass@64": 0.9652302587641902, |
| "pass@128": 0.977491961414791 |
| }, |
| "11": { |
| "pass@1": 0.7507688492063492, |
| "pass@2": 0.8152867610298712, |
| "pass@4": 0.8574687646187082, |
| "pass@8": 0.8888661836516896, |
| "pass@16": 0.9135599558410991, |
| "pass@32": 0.9314423496372826, |
| "pass@64": 0.9434417545138772, |
| "pass@128": 0.9523809523809523 |
| }, |
| "12": { |
| "pass@1": 0.7076404816513762, |
| "pass@2": 0.8042358201497741, |
| "pass@4": 0.8671578274619333, |
| "pass@8": 0.9085258038760612, |
| "pass@16": 0.935189808714328, |
| "pass@32": 0.9534243395021395, |
| "pass@64": 0.9653087455773289, |
| "pass@128": 0.9724770642201835 |
| }, |
| "13": { |
| "pass@1": 0.5027488425925926, |
| "pass@2": 0.6106045555069503, |
| "pass@4": 0.6959111670647646, |
| "pass@8": 0.7622771166009414, |
| "pass@16": 0.8123751233047263, |
| "pass@32": 0.8480814715984645, |
| "pass@64": 0.8730092960227223, |
| "pass@128": 0.8919753086419753 |
| }, |
| "14": { |
| "pass@1": 0.3861044139465875, |
| "pass@2": 0.4967668637117687, |
| "pass@4": 0.5935214991672034, |
| "pass@8": 0.6672018859137244, |
| "pass@16": 0.71946447521043, |
| "pass@32": 0.7584769248935936, |
| "pass@64": 0.7924551518791305, |
| "pass@128": 0.827893175074184 |
| }, |
| "15": { |
| "pass@1": 0.24127121913580246, |
| "pass@2": 0.32664900177408374, |
| "pass@4": 0.4125610458993551, |
| "pass@8": 0.49270826498349163, |
| "pass@16": 0.5618038483266853, |
| "pass@32": 0.6195859693933924, |
| "pass@64": 0.6711680233603406, |
| "pass@128": 0.7191358024691358 |
| }, |
| "16": { |
| "pass@1": 0.19690265486725664, |
| "pass@2": 0.2587497532111583, |
| "pass@4": 0.3189182358842312, |
| "pass@8": 0.374695087684852, |
| "pass@16": 0.4241129705890384, |
| "pass@32": 0.4690045235965585, |
| "pass@64": 0.5130468430129036, |
| "pass@128": 0.5634218289085545 |
| }, |
| "17": { |
| "pass@1": 0.17613422439759036, |
| "pass@2": 0.23685675279859578, |
| "pass@4": 0.2954108272234041, |
| "pass@8": 0.34783661316669684, |
| "pass@16": 0.3928556096830005, |
| "pass@32": 0.43333668541543363, |
| "pass@64": 0.4700079164554424, |
| "pass@128": 0.5060240963855421 |
| }, |
| "18": { |
| "pass@1": 0.1748991935483871, |
| "pass@2": 0.23574501524003058, |
| "pass@4": 0.2958681713172949, |
| "pass@8": 0.34993805557826135, |
| "pass@16": 0.395412438432603, |
| "pass@32": 0.43308274288586, |
| "pass@64": 0.4660027797221985, |
| "pass@128": 0.4935483870967742 |
| }, |
| "19": { |
| "pass@1": 0.15717050604229607, |
| "pass@2": 0.21232374158479428, |
| "pass@4": 0.2676054354610507, |
| "pass@8": 0.31955934750096615, |
| "pass@16": 0.3666705109285142, |
| "pass@32": 0.4077764639729569, |
| "pass@64": 0.44331074246388014, |
| "pass@128": 0.4773413897280967 |
| }, |
| "20": { |
| "pass@1": 0.14761636636636635, |
| "pass@2": 0.20476615887541083, |
| "pass@4": 0.2636930164134889, |
| "pass@8": 0.32053535130427724, |
| "pass@16": 0.3732904435639585, |
| "pass@32": 0.4200946391152685, |
| "pass@64": 0.4584333411955923, |
| "pass@128": 0.4864864864864865 |
| } |
| } |
| } |
| } |
| } |
| ] |
