| [ |
| { |
| "checkpoint": "checkpoint-21350", |
| "id_acc": 0.9463585069444445, |
| "ood_acc": 0.3788546875, |
| "total_acc": 0.6476722861842106, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 197.93385069444443, |
| "ood_avg_resp_len": 316.5756515625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 525646, |
| "answer_accuracy": 0.6491636697755295, |
| "avg_response_len": 248.8114650845716, |
| "resp_tokens_sum": 201469610, |
| "pass_at_k": { |
| "pass@1": 0.6491636697755295, |
| "pass@2": 0.6918507562216686, |
| "pass@4": 0.7265823497333529, |
| "pass@8": 0.7552785357582993, |
| "pass@16": 0.779610731297473, |
| "pass@32": 0.8004989331569241, |
| "pass@64": 0.8181713588791755, |
| "pass@128": 0.8330698703762252 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8773276179941003, |
| "pass@2": 0.9119181590133094, |
| "pass@4": 0.9311756113450421, |
| "pass@8": 0.9420426122301978, |
| "pass@16": 0.9502590673394334, |
| "pass@32": 0.9584931653088372, |
| "pass@64": 0.9669784816191926, |
| "pass@128": 0.9734513274336283 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9973241590214067, |
| "pass@2": 0.9977444225721784, |
| "pass@4": 0.9983450506186725, |
| "pass@8": 0.9991236482754671, |
| "pass@16": 0.9997712860612012, |
| "pass@32": 0.9999889701449637, |
| "pass@64": 0.9999999970955962, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9881591796875, |
| "pass@2": 0.9902486005167324, |
| "pass@4": 0.9916557953693287, |
| "pass@8": 0.9928006480587112, |
| "pass@16": 0.9937693424773156, |
| "pass@32": 0.994500319642821, |
| "pass@64": 0.9953124622828344, |
| "pass@128": 0.996875 |
| }, |
| "5": { |
| "pass@1": 0.9253224206349207, |
| "pass@2": 0.9414018560179978, |
| "pass@4": 0.9528689018039411, |
| "pass@8": 0.9625002308833911, |
| "pass@16": 0.9709636195225438, |
| "pass@32": 0.9777918358496315, |
| "pass@64": 0.9838997800481912, |
| "pass@128": 0.9904761904761905 |
| }, |
| "6": { |
| "pass@1": 0.9671594311377245, |
| "pass@2": 0.9752820883822906, |
| "pass@4": 0.9815461927663233, |
| "pass@8": 0.9861414925202345, |
| "pass@16": 0.9886085746496264, |
| "pass@32": 0.9900551906526195, |
| "pass@64": 0.9917723246221735, |
| "pass@128": 0.9940119760479041 |
| }, |
| "7": { |
| "pass@1": 0.9505593565088757, |
| "pass@2": 0.9665288810977031, |
| "pass@4": 0.976226311115696, |
| "pass@8": 0.9825814576886557, |
| "pass@16": 0.9870875795976058, |
| "pass@32": 0.9910536770529155, |
| "pass@64": 0.9947454835702054, |
| "pass@128": 0.9970414201183432 |
| }, |
| "8": { |
| "pass@1": 0.9407301682692307, |
| "pass@2": 0.9576164382697357, |
| "pass@4": 0.9672972564126601, |
| "pass@8": 0.9735731917238386, |
| "pass@16": 0.977776925424273, |
| "pass@32": 0.980567025073276, |
| "pass@64": 0.9823063355516893, |
| "pass@128": 0.9839743589743589 |
| }, |
| "9": { |
| "pass@1": 0.8992077464788732, |
| "pass@2": 0.9294131917489195, |
| "pass@4": 0.9485870498582041, |
| "pass@8": 0.9623563850919464, |
| "pass@16": 0.9729279398669448, |
| "pass@32": 0.9810885217868446, |
| "pass@64": 0.9864213997414775, |
| "pass@128": 0.9887323943661972 |
| }, |
| "11": { |
| "pass@1": 0.8281471317280453, |
| "pass@2": 0.8772393126408069, |
| "pass@4": 0.9072971293779494, |
| "pass@8": 0.9263560752155664, |
| "pass@16": 0.9417339425971359, |
| "pass@32": 0.9566709407386268, |
| "pass@64": 0.9699104428716377, |
| "pass@128": 0.9801699716713881 |
| }, |
| "12": { |
| "pass@1": 0.7567973163841808, |
| "pass@2": 0.8258520452422264, |
| "pass@4": 0.8681805065680348, |
| "pass@8": 0.8951861291592782, |
| "pass@16": 0.9141290331047311, |
| "pass@32": 0.9301895220722955, |
| "pass@64": 0.9461462846873937, |
| "pass@128": 0.96045197740113 |
| }, |
| "13": { |
| "pass@1": 0.5737656739811913, |
| "pass@2": 0.6663702083281907, |
| "pass@4": 0.7377947493044555, |
| "pass@8": 0.7931345691897921, |
| "pass@16": 0.8379584677978166, |
| "pass@32": 0.8751047840970735, |
| "pass@64": 0.9032302694399705, |
| "pass@128": 0.9216300940438872 |
| }, |
| "14": { |
| "pass@1": 0.45030179127725856, |
| "pass@2": 0.5457960789363944, |
| "pass@4": 0.6231636244067621, |
| "pass@8": 0.6869051629447177, |
| "pass@16": 0.7392870707923059, |
| "pass@32": 0.7804201059407638, |
| "pass@64": 0.8141525302007103, |
| "pass@128": 0.8473520249221184 |
| }, |
| "15": { |
| "pass@1": 0.3126354768786127, |
| "pass@2": 0.3937431017022439, |
| "pass@4": 0.4662351267688358, |
| "pass@8": 0.5297946130837053, |
| "pass@16": 0.586091346781155, |
| "pass@32": 0.6351826501037516, |
| "pass@64": 0.6761407868785784, |
| "pass@128": 0.7109826589595376 |
| }, |
| "16": { |
| "pass@1": 0.25042842741935484, |
| "pass@2": 0.32574811087122146, |
| "pass@4": 0.39351368578927626, |
| "pass@8": 0.45303587701396264, |
| "pass@16": 0.5062086173550845, |
| "pass@32": 0.5539039754509739, |
| "pass@64": 0.5955508562334999, |
| "pass@128": 0.632258064516129 |
| }, |
| "17": { |
| "pass@1": 0.16848807803468208, |
| "pass@2": 0.22524364503208774, |
| "pass@4": 0.2802942645175134, |
| "pass@8": 0.330783163112241, |
| "pass@16": 0.37907349032212845, |
| "pass@32": 0.42469351653658793, |
| "pass@64": 0.4638600016363443, |
| "pass@128": 0.4913294797687861 |
| }, |
| "18": { |
| "pass@1": 0.1693080357142857, |
| "pass@2": 0.22570620078740164, |
| "pass@4": 0.28333418795864795, |
| "pass@8": 0.33963163946315195, |
| "pass@16": 0.3904539856568932, |
| "pass@32": 0.4325615854877157, |
| "pass@64": 0.464574906335151, |
| "pass@128": 0.49142857142857144 |
| }, |
| "19": { |
| "pass@1": 0.16042605105105104, |
| "pass@2": 0.21579022427939754, |
| "pass@4": 0.2713904286738932, |
| "pass@8": 0.320673574822643, |
| "pass@16": 0.36145502547813574, |
| "pass@32": 0.3960321250148641, |
| "pass@64": 0.42531332447168435, |
| "pass@128": 0.45045045045045046 |
| }, |
| "20": { |
| "pass@1": 0.15836148648648649, |
| "pass@2": 0.20984591875339906, |
| "pass@4": 0.25997215860404826, |
| "pass@8": 0.3057199370131288, |
| "pass@16": 0.34552328217296413, |
| "pass@32": 0.38000113840819033, |
| "pass@64": 0.41232314729781633, |
| "pass@128": 0.4444444444444444 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 532584, |
| "answer_accuracy": 0.6486067809820732, |
| "avg_response_len": 257.71719967848793, |
| "resp_tokens_sum": 211616747, |
| "pass_at_k": { |
| "pass@1": 0.6486067809820732, |
| "pass@2": 0.6908541473600844, |
| "pass@4": 0.7264055504607114, |
| "pass@8": 0.7563388816938654, |
| "pass@16": 0.7814218557464435, |
| "pass@32": 0.8029980862811904, |
| "pass@64": 0.821935053724575, |
| "pass@128": 0.8388152766952455 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8393884202453987, |
| "pass@2": 0.8698545662045313, |
| "pass@4": 0.8904956702114691, |
| "pass@8": 0.9055833604644317, |
| "pass@16": 0.9173558658878211, |
| "pass@32": 0.9274155966730994, |
| "pass@64": 0.9375057974027622, |
| "pass@128": 0.9478527607361963 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9979876893939394, |
| "pass@2": 0.9995783375089478, |
| "pass@4": 0.9999699469384509, |
| "pass@8": 0.9999997974031156, |
| "pass@16": 0.9999999999966537, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9895062869822485, |
| "pass@2": 0.9918151295252295, |
| "pass@4": 0.9931505192768064, |
| "pass@8": 0.9945047565814593, |
| "pass@16": 0.9959301126815046, |
| "pass@32": 0.9968252000189942, |
| "pass@64": 0.9970371391134153, |
| "pass@128": 0.9970414201183432 |
| }, |
| "5": { |
| "pass@1": 0.9276947463768116, |
| "pass@2": 0.941486933698505, |
| "pass@4": 0.9521146527879665, |
| "pass@8": 0.9596253376638073, |
| "pass@16": 0.9648710403519182, |
| "pass@32": 0.9689627240711924, |
| "pass@64": 0.9727993021286935, |
| "pass@128": 0.9768115942028985 |
| }, |
| "6": { |
| "pass@1": 0.973047994269341, |
| "pass@2": 0.9776248364280394, |
| "pass@4": 0.9810814086097404, |
| "pass@8": 0.9838584328812273, |
| "pass@16": 0.9865986488348304, |
| "pass@32": 0.9898077163259399, |
| "pass@64": 0.9933483043882753, |
| "pass@128": 0.997134670487106 |
| }, |
| "7": { |
| "pass@1": 0.949534862716763, |
| "pass@2": 0.9634724466341997, |
| "pass@4": 0.9741216728328035, |
| "pass@8": 0.9818345949858238, |
| "pass@16": 0.9860409160803892, |
| "pass@32": 0.9882156768256282, |
| "pass@64": 0.9906125875539492, |
| "pass@128": 0.9942196531791907 |
| }, |
| "8": { |
| "pass@1": 0.9240096830985915, |
| "pass@2": 0.94706491904181, |
| "pass@4": 0.9618155671562175, |
| "pass@8": 0.972741330632731, |
| "pass@16": 0.9807618398300412, |
| "pass@32": 0.9868507906518273, |
| "pass@64": 0.9914611399873605, |
| "pass@128": 0.9943661971830986 |
| }, |
| "9": { |
| "pass@1": 0.8909758607784432, |
| "pass@2": 0.918286458578905, |
| "pass@4": 0.9358988911939895, |
| "pass@8": 0.9502432948194586, |
| "pass@16": 0.9635661425413976, |
| "pass@32": 0.9753201066438478, |
| "pass@64": 0.9822986698173887, |
| "pass@128": 0.9850299401197605 |
| }, |
| "11": { |
| "pass@1": 0.8322195030120482, |
| "pass@2": 0.8740064836116118, |
| "pass@4": 0.9007651367562485, |
| "pass@8": 0.9202482006306049, |
| "pass@16": 0.9352773543837686, |
| "pass@32": 0.9477013646227804, |
| "pass@64": 0.9572351213137299, |
| "pass@128": 0.963855421686747 |
| }, |
| "12": { |
| "pass@1": 0.7488489420062696, |
| "pass@2": 0.8215862038111226, |
| "pass@4": 0.8709659145271413, |
| "pass@8": 0.9032511169652427, |
| "pass@16": 0.9243224931700824, |
| "pass@32": 0.9391710642481247, |
| "pass@64": 0.9510478149135736, |
| "pass@128": 0.9592476489028213 |
| }, |
| "13": { |
| "pass@1": 0.5842086834733894, |
| "pass@2": 0.676471966739452, |
| "pass@4": 0.741411255840919, |
| "pass@8": 0.7904271208733067, |
| "pass@16": 0.8313294364350797, |
| "pass@32": 0.8672863371588099, |
| "pass@64": 0.8973304964641204, |
| "pass@128": 0.9215686274509803 |
| }, |
| "14": { |
| "pass@1": 0.45575201023391815, |
| "pass@2": 0.5523883938849747, |
| "pass@4": 0.6318049690060672, |
| "pass@8": 0.6928149400614, |
| "pass@16": 0.7359096694993563, |
| "pass@32": 0.7666387497632563, |
| "pass@64": 0.790515190000481, |
| "pass@128": 0.8099415204678363 |
| }, |
| "15": { |
| "pass@1": 0.3084280303030303, |
| "pass@2": 0.3932630935337627, |
| "pass@4": 0.471590225085501, |
| "pass@8": 0.5436062297147697, |
| "pass@16": 0.6081874040818871, |
| "pass@32": 0.6620940150440326, |
| "pass@64": 0.7063833432601488, |
| "pass@128": 0.7454545454545455 |
| }, |
| "16": { |
| "pass@1": 0.22760861823361822, |
| "pass@2": 0.3023122209884019, |
| "pass@4": 0.3737860908412092, |
| "pass@8": 0.43597528456928797, |
| "pass@16": 0.4874427513369775, |
| "pass@32": 0.532987304252585, |
| "pass@64": 0.5746519986523746, |
| "pass@128": 0.6096866096866097 |
| }, |
| "17": { |
| "pass@1": 0.19157608695652173, |
| "pass@2": 0.2501818726463538, |
| "pass@4": 0.3103001718147032, |
| "pass@8": 0.36859091329527444, |
| "pass@16": 0.4233694246594647, |
| "pass@32": 0.47383565180387, |
| "pass@64": 0.5219014440208865, |
| "pass@128": 0.5714285714285714 |
| }, |
| "18": { |
| "pass@1": 0.1758501838235294, |
| "pass@2": 0.2325841680176007, |
| "pass@4": 0.2908814795209421, |
| "pass@8": 0.3483809368179013, |
| "pass@16": 0.4025550346930342, |
| "pass@32": 0.45392796747630026, |
| "pass@64": 0.5037675550562966, |
| "pass@128": 0.5529411764705883 |
| }, |
| "19": { |
| "pass@1": 0.14306640625, |
| "pass@2": 0.19530993684383194, |
| "pass@4": 0.2483651784040386, |
| "pass@8": 0.29777947891872014, |
| "pass@16": 0.34304739154710734, |
| "pass@32": 0.3863642388713654, |
| "pass@64": 0.4263632707832371, |
| "pass@128": 0.46130952380952384 |
| }, |
| "20": { |
| "pass@1": 0.14897548652694612, |
| "pass@2": 0.20376631088217265, |
| "pass@4": 0.25812340644919385, |
| "pass@8": 0.30629608870839276, |
| "pass@16": 0.3463328241220228, |
| "pass@32": 0.3798774817729502, |
| "pass@64": 0.40926661845536116, |
| "pass@128": 0.437125748502994 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 516909, |
| "answer_accuracy": 0.6452071517015497, |
| "avg_response_len": 274.7921405675827, |
| "resp_tokens_sum": 220150273, |
| "pass_at_k": { |
| "pass@1": 0.6452071517015497, |
| "pass@2": 0.6865093792497867, |
| "pass@4": 0.7209135034376871, |
| "pass@8": 0.7497234395452865, |
| "pass@16": 0.7733847971497279, |
| "pass@32": 0.7930687594794426, |
| "pass@64": 0.810047360600001, |
| "pass@128": 0.8256910049528678 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8871035447761194, |
| "pass@2": 0.917110559407686, |
| "pass@4": 0.9365289956292769, |
| "pass@8": 0.95085081510218, |
| "pass@16": 0.9616873600439593, |
| "pass@32": 0.9704475586213182, |
| "pass@64": 0.9780345237152435, |
| "pass@128": 0.982089552238806 |
| }, |
| "2": { |
| "pass@1": 0.9999776785714286, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9995444606413995, |
| "pass@2": 0.9999508591400565, |
| "pass@4": 0.9999993495710995, |
| "pass@8": 0.9999999999504271, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9823419225146199, |
| "pass@2": 0.9854621379564396, |
| "pass@4": 0.9880769901021142, |
| "pass@8": 0.9902890222952649, |
| "pass@16": 0.9922729187814656, |
| "pass@32": 0.9941339389256002, |
| "pass@64": 0.9959935170640539, |
| "pass@128": 0.9970760233918129 |
| }, |
| "5": { |
| "pass@1": 0.9192095588235294, |
| "pass@2": 0.9368833950903197, |
| "pass@4": 0.9486726628840507, |
| "pass@8": 0.957187841731995, |
| "pass@16": 0.963640097381688, |
| "pass@32": 0.9699201940179724, |
| "pass@64": 0.9774210870046359, |
| "pass@128": 0.9852941176470589 |
| }, |
| "6": { |
| "pass@1": 0.9850897082018928, |
| "pass@2": 0.9905075573163766, |
| "pass@4": 0.9929501176311952, |
| "pass@8": 0.9936089564592303, |
| "pass@16": 0.9936894407460112, |
| "pass@32": 0.9936908516082206, |
| "pass@64": 0.9936908517350158, |
| "pass@128": 0.9936908517350158 |
| }, |
| "7": { |
| "pass@1": 0.9513202136075949, |
| "pass@2": 0.9653273105003486, |
| "pass@4": 0.9741560914458472, |
| "pass@8": 0.9809422762648561, |
| "pass@16": 0.9865958409360319, |
| "pass@32": 0.9910348161676964, |
| "pass@64": 0.9942570067837525, |
| "pass@128": 0.9968354430379747 |
| }, |
| "8": { |
| "pass@1": 0.9035285285285285, |
| "pass@2": 0.9250335473268545, |
| "pass@4": 0.9401216576531538, |
| "pass@8": 0.9530895360198267, |
| "pass@16": 0.9647251870543512, |
| "pass@32": 0.9739953866287852, |
| "pass@64": 0.9797902253746495, |
| "pass@128": 0.9819819819819819 |
| }, |
| "9": { |
| "pass@1": 0.8857013665594855, |
| "pass@2": 0.9113160493202016, |
| "pass@4": 0.9282795167083213, |
| "pass@8": 0.9412904877973327, |
| "pass@16": 0.9506791980302753, |
| "pass@32": 0.9574335414534275, |
| "pass@64": 0.962500672127828, |
| "pass@128": 0.9678456591639871 |
| }, |
| "11": { |
| "pass@1": 0.8270337301587302, |
| "pass@2": 0.8687472659667539, |
| "pass@4": 0.8942530689616175, |
| "pass@8": 0.913000742277685, |
| "pass@16": 0.9287522778885744, |
| "pass@32": 0.9423010463849473, |
| "pass@64": 0.9536259185529427, |
| "pass@128": 0.9619047619047619 |
| }, |
| "12": { |
| "pass@1": 0.7741303516819572, |
| "pass@2": 0.844897917720147, |
| "pass@4": 0.8911836662619007, |
| "pass@8": 0.9227577810625305, |
| "pass@16": 0.9434655328339684, |
| "pass@32": 0.9580731530824094, |
| "pass@64": 0.9686499242431075, |
| "pass@128": 0.9755351681957186 |
| }, |
| "13": { |
| "pass@1": 0.5628134645061729, |
| "pass@2": 0.6580126919898899, |
| "pass@4": 0.7298308666277817, |
| "pass@8": 0.7849102761957668, |
| "pass@16": 0.8272916207877943, |
| "pass@32": 0.8597796191468667, |
| "pass@64": 0.8834876179969318, |
| "pass@128": 0.9012345679012346 |
| }, |
| "14": { |
| "pass@1": 0.42303412462908013, |
| "pass@2": 0.5254594879553262, |
| "pass@4": 0.612024243167675, |
| "pass@8": 0.6795870113254974, |
| "pass@16": 0.7299641829436971, |
| "pass@32": 0.769675569678991, |
| "pass@64": 0.802551172129445, |
| "pass@128": 0.8308605341246291 |
| }, |
| "15": { |
| "pass@1": 0.2721113040123457, |
| "pass@2": 0.3505919181977249, |
| "pass@4": 0.42352521617668165, |
| "pass@8": 0.49062896925691024, |
| "pass@16": 0.5509084623861222, |
| "pass@32": 0.6026686433028146, |
| "pass@64": 0.648546132335972, |
| "pass@128": 0.6975308641975309 |
| }, |
| "16": { |
| "pass@1": 0.1982853982300885, |
| "pass@2": 0.25889056802081145, |
| "pass@4": 0.3179625279140993, |
| "pass@8": 0.3723615276522692, |
| "pass@16": 0.41992990025116966, |
| "pass@32": 0.46214547052895755, |
| "pass@64": 0.5066410316005072, |
| "pass@128": 0.5634218289085545 |
| }, |
| "17": { |
| "pass@1": 0.17211031626506024, |
| "pass@2": 0.2284550356939569, |
| "pass@4": 0.2824050603426078, |
| "pass@8": 0.3306790083684093, |
| "pass@16": 0.3695724116618554, |
| "pass@32": 0.40174054863361564, |
| "pass@64": 0.4304838558018472, |
| "pass@128": 0.4578313253012048 |
| }, |
| "18": { |
| "pass@1": 0.18351814516129034, |
| "pass@2": 0.2443234855219709, |
| "pass@4": 0.3019939807927236, |
| "pass@8": 0.35022866682181714, |
| "pass@16": 0.38934737936610586, |
| "pass@32": 0.4241763498467773, |
| "pass@64": 0.457008988079855, |
| "pass@128": 0.4870967741935484 |
| }, |
| "19": { |
| "pass@1": 0.15853946374622357, |
| "pass@2": 0.21041210719128384, |
| "pass@4": 0.26191707786904267, |
| "pass@8": 0.3102665481682076, |
| "pass@16": 0.35281055902223907, |
| "pass@32": 0.38815261030610226, |
| "pass@64": 0.4175298687617645, |
| "pass@128": 0.44410876132930516 |
| }, |
| "20": { |
| "pass@1": 0.15064283033033032, |
| "pass@2": 0.20356281478328725, |
| "pass@4": 0.2574440165700009, |
| "pass@8": 0.30892386841405955, |
| "pass@16": 0.35627459223670677, |
| "pass@32": 0.3974007494071951, |
| "pass@64": 0.4299470015533774, |
| "pass@128": 0.45345345345345345 |
| } |
| } |
| } |
| } |
| } |
| ] |
